?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
net: reduce the unnecessary memory allocation of multiqueue
[qemu/rayw.git] / exec.c
bloba41bcb86943821496e21158dd11bfc93c25b92e5
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "hw/xen.h"
35 #include "qemu/timer.h"
36 #include "qemu/config-file.h"
37 #include "exec/memory.h"
38 #include "sysemu/dma.h"
39 #include "exec/address-spaces.h"
40 #if defined(CONFIG_USER_ONLY)
41 #include <qemu.h>
42 #else /* !CONFIG_USER_ONLY */
43 #include "sysemu/xen-mapcache.h"
44 #include "trace.h"
45 #endif
46 #include "exec/cpu-all.h"
47
48 #include "exec/cputlb.h"
49 #include "translate-all.h"
50
51 #include "exec/memory-internal.h"
52
53 //#define DEBUG_UNASSIGNED
54 //#define DEBUG_SUBPAGE
55
56 #if !defined(CONFIG_USER_ONLY)
57 int phys_ram_fd;
58 static int in_migration;
59
60 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
61
62 static MemoryRegion *system_memory;
63 static MemoryRegion *system_io;
64
65 AddressSpace address_space_io;
66 AddressSpace address_space_memory;
67 DMAContext dma_context_memory;
68
69 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
70 static MemoryRegion io_mem_subpage_ram;
71
72 #endif
73
74 CPUArchState *first_cpu;
75 /* current CPU in the current thread. It is only valid inside
76 cpu_exec() */
77 DEFINE_TLS(CPUArchState *,cpu_single_env);
78 /* 0 = Do not count executed instructions.
79 1 = Precise instruction counting.
80 2 = Adaptive rate instruction counting. */
81 int use_icount;
82
83 #if !defined(CONFIG_USER_ONLY)
84
85 static MemoryRegionSection *phys_sections;
86 static unsigned phys_sections_nb, phys_sections_nb_alloc;
87 static uint16_t phys_section_unassigned;
88 static uint16_t phys_section_notdirty;
89 static uint16_t phys_section_rom;
90 static uint16_t phys_section_watch;
91
92 /* Simple allocator for PhysPageEntry nodes */
93 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
94 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
95
96 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
97
98 static void io_mem_init(void);
99 static void memory_map_init(void);
100 static void *qemu_safe_ram_ptr(ram_addr_t addr);
101
102 static MemoryRegion io_mem_watch;
103 #endif
104
105 #if !defined(CONFIG_USER_ONLY)
106
107 static void phys_map_node_reserve(unsigned nodes)
108 {
109 if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
110 typedef PhysPageEntry Node[L2_SIZE];
111 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
112 phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
113 phys_map_nodes_nb + nodes);
114 phys_map_nodes = g_renew(Node, phys_map_nodes,
115 phys_map_nodes_nb_alloc);
116 }
117 }
118
119 static uint16_t phys_map_node_alloc(void)
120 {
121 unsigned i;
122 uint16_t ret;
123
124 ret = phys_map_nodes_nb++;
125 assert(ret != PHYS_MAP_NODE_NIL);
126 assert(ret != phys_map_nodes_nb_alloc);
127 for (i = 0; i < L2_SIZE; ++i) {
128 phys_map_nodes[ret][i].is_leaf = 0;
129 phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
130 }
131 return ret;
132 }
133
134 static void phys_map_nodes_reset(void)
135 {
136 phys_map_nodes_nb = 0;
137 }
138
139
140 static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
141 hwaddr *nb, uint16_t leaf,
142 int level)
143 {
144 PhysPageEntry *p;
145 int i;
146 hwaddr step = (hwaddr)1 << (level * L2_BITS);
147
148 if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
149 lp->ptr = phys_map_node_alloc();
150 p = phys_map_nodes[lp->ptr];
151 if (level == 0) {
152 for (i = 0; i < L2_SIZE; i++) {
153 p[i].is_leaf = 1;
154 p[i].ptr = phys_section_unassigned;
155 }
156 }
157 } else {
158 p = phys_map_nodes[lp->ptr];
159 }
160 lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
161
162 while (*nb && lp < &p[L2_SIZE]) {
163 if ((*index & (step - 1)) == 0 && *nb >= step) {
164 lp->is_leaf = true;
165 lp->ptr = leaf;
166 *index += step;
167 *nb -= step;
168 } else {
169 phys_page_set_level(lp, index, nb, leaf, level - 1);
170 }
171 ++lp;
172 }
173 }
174
175 static void phys_page_set(AddressSpaceDispatch *d,
176 hwaddr index, hwaddr nb,
177 uint16_t leaf)
178 {
179 /* Wildly overreserve - it doesn't matter much. */
180 phys_map_node_reserve(3 * P_L2_LEVELS);
181
182 phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
183 }
184
185 MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
186 {
187 PhysPageEntry lp = d->phys_map;
188 PhysPageEntry *p;
189 int i;
190 uint16_t s_index = phys_section_unassigned;
191
192 for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
193 if (lp.ptr == PHYS_MAP_NODE_NIL) {
194 goto not_found;
195 }
196 p = phys_map_nodes[lp.ptr];
197 lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
198 }
199
200 s_index = lp.ptr;
201 not_found:
202 return &phys_sections[s_index];
203 }
204
205 bool memory_region_is_unassigned(MemoryRegion *mr)
206 {
207 return mr != &io_mem_ram && mr != &io_mem_rom
208 && mr != &io_mem_notdirty && !mr->rom_device
209 && mr != &io_mem_watch;
210 }
211 #endif
212
213 void cpu_exec_init_all(void)
214 {
215 #if !defined(CONFIG_USER_ONLY)
216 qemu_mutex_init(&ram_list.mutex);
217 memory_map_init();
218 io_mem_init();
219 #endif
220 }
221
222 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
223
224 static int cpu_common_post_load(void *opaque, int version_id)
225 {
226 CPUArchState *env = opaque;
227
228 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
229 version_id is increased. */
230 env->interrupt_request &= ~0x01;
231 tlb_flush(env, 1);
232
233 return 0;
234 }
235
236 static const VMStateDescription vmstate_cpu_common = {
237 .name = "cpu_common",
238 .version_id = 1,
239 .minimum_version_id = 1,
240 .minimum_version_id_old = 1,
241 .post_load = cpu_common_post_load,
242 .fields = (VMStateField []) {
243 VMSTATE_UINT32(halted, CPUArchState),
244 VMSTATE_UINT32(interrupt_request, CPUArchState),
245 VMSTATE_END_OF_LIST()
246 }
247 };
248 #endif
249
250 CPUState *qemu_get_cpu(int index)
251 {
252 CPUArchState *env = first_cpu;
253 CPUState *cpu = NULL;
254
255 while (env) {
256 cpu = ENV_GET_CPU(env);
257 if (cpu->cpu_index == index) {
258 break;
259 }
260 env = env->next_cpu;
261 }
262
263 return cpu;
264 }
265
266 void cpu_exec_init(CPUArchState *env)
267 {
268 CPUState *cpu = ENV_GET_CPU(env);
269 CPUArchState **penv;
270 int cpu_index;
271
272 #if defined(CONFIG_USER_ONLY)
273 cpu_list_lock();
274 #endif
275 env->next_cpu = NULL;
276 penv = &first_cpu;
277 cpu_index = 0;
278 while (*penv != NULL) {
279 penv = &(*penv)->next_cpu;
280 cpu_index++;
281 }
282 cpu->cpu_index = cpu_index;
283 cpu->numa_node = 0;
284 QTAILQ_INIT(&env->breakpoints);
285 QTAILQ_INIT(&env->watchpoints);
286 #ifndef CONFIG_USER_ONLY
287 cpu->thread_id = qemu_get_thread_id();
288 #endif
289 *penv = env;
290 #if defined(CONFIG_USER_ONLY)
291 cpu_list_unlock();
292 #endif
293 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
294 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
295 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
296 cpu_save, cpu_load, env);
297 #endif
298 }
299
300 #if defined(TARGET_HAS_ICE)
301 #if defined(CONFIG_USER_ONLY)
302 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
303 {
304 tb_invalidate_phys_page_range(pc, pc + 1, 0);
305 }
306 #else
307 static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
308 {
309 tb_invalidate_phys_addr(cpu_get_phys_page_debug(env, pc) |
310 (pc & ~TARGET_PAGE_MASK));
311 }
312 #endif
313 #endif /* TARGET_HAS_ICE */
314
315 #if defined(CONFIG_USER_ONLY)
316 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
317
318 {
319 }
320
321 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
322 int flags, CPUWatchpoint **watchpoint)
323 {
324 return -ENOSYS;
325 }
326 #else
327 /* Add a watchpoint. */
328 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
329 int flags, CPUWatchpoint **watchpoint)
330 {
331 target_ulong len_mask = ~(len - 1);
332 CPUWatchpoint *wp;
333
334 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
335 if ((len & (len - 1)) || (addr & ~len_mask) ||
336 len == 0 || len > TARGET_PAGE_SIZE) {
337 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
338 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
339 return -EINVAL;
340 }
341 wp = g_malloc(sizeof(*wp));
342
343 wp->vaddr = addr;
344 wp->len_mask = len_mask;
345 wp->flags = flags;
346
347 /* keep all GDB-injected watchpoints in front */
348 if (flags & BP_GDB)
349 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
350 else
351 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
352
353 tlb_flush_page(env, addr);
354
355 if (watchpoint)
356 *watchpoint = wp;
357 return 0;
358 }
359
360 /* Remove a specific watchpoint. */
361 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
362 int flags)
363 {
364 target_ulong len_mask = ~(len - 1);
365 CPUWatchpoint *wp;
366
367 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
368 if (addr == wp->vaddr && len_mask == wp->len_mask
369 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
370 cpu_watchpoint_remove_by_ref(env, wp);
371 return 0;
372 }
373 }
374 return -ENOENT;
375 }
376
377 /* Remove a specific watchpoint by reference. */
378 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
379 {
380 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
381
382 tlb_flush_page(env, watchpoint->vaddr);
383
384 g_free(watchpoint);
385 }
386
387 /* Remove all matching watchpoints. */
388 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
389 {
390 CPUWatchpoint *wp, *next;
391
392 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
393 if (wp->flags & mask)
394 cpu_watchpoint_remove_by_ref(env, wp);
395 }
396 }
397 #endif
398
399 /* Add a breakpoint. */
400 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
401 CPUBreakpoint **breakpoint)
402 {
403 #if defined(TARGET_HAS_ICE)
404 CPUBreakpoint *bp;
405
406 bp = g_malloc(sizeof(*bp));
407
408 bp->pc = pc;
409 bp->flags = flags;
410
411 /* keep all GDB-injected breakpoints in front */
412 if (flags & BP_GDB)
413 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
414 else
415 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
416
417 breakpoint_invalidate(env, pc);
418
419 if (breakpoint)
420 *breakpoint = bp;
421 return 0;
422 #else
423 return -ENOSYS;
424 #endif
425 }
426
427 /* Remove a specific breakpoint. */
428 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
429 {
430 #if defined(TARGET_HAS_ICE)
431 CPUBreakpoint *bp;
432
433 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
434 if (bp->pc == pc && bp->flags == flags) {
435 cpu_breakpoint_remove_by_ref(env, bp);
436 return 0;
437 }
438 }
439 return -ENOENT;
440 #else
441 return -ENOSYS;
442 #endif
443 }
444
445 /* Remove a specific breakpoint by reference. */
446 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
447 {
448 #if defined(TARGET_HAS_ICE)
449 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
450
451 breakpoint_invalidate(env, breakpoint->pc);
452
453 g_free(breakpoint);
454 #endif
455 }
456
457 /* Remove all matching breakpoints. */
458 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
459 {
460 #if defined(TARGET_HAS_ICE)
461 CPUBreakpoint *bp, *next;
462
463 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
464 if (bp->flags & mask)
465 cpu_breakpoint_remove_by_ref(env, bp);
466 }
467 #endif
468 }
469
470 /* enable or disable single step mode. EXCP_DEBUG is returned by the
471 CPU loop after each instruction */
472 void cpu_single_step(CPUArchState *env, int enabled)
473 {
474 #if defined(TARGET_HAS_ICE)
475 if (env->singlestep_enabled != enabled) {
476 env->singlestep_enabled = enabled;
477 if (kvm_enabled())
478 kvm_update_guest_debug(env, 0);
479 else {
480 /* must flush all the translated code to avoid inconsistencies */
481 /* XXX: only flush what is necessary */
482 tb_flush(env);
483 }
484 }
485 #endif
486 }
487
488 void cpu_reset_interrupt(CPUArchState *env, int mask)
489 {
490 env->interrupt_request &= ~mask;
491 }
492
493 void cpu_exit(CPUArchState *env)
494 {
495 CPUState *cpu = ENV_GET_CPU(env);
496
497 cpu->exit_request = 1;
498 cpu_unlink_tb(cpu);
499 }
500
501 void cpu_abort(CPUArchState *env, const char *fmt, ...)
502 {
503 va_list ap;
504 va_list ap2;
505
506 va_start(ap, fmt);
507 va_copy(ap2, ap);
508 fprintf(stderr, "qemu: fatal: ");
509 vfprintf(stderr, fmt, ap);
510 fprintf(stderr, "\n");
511 cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
512 if (qemu_log_enabled()) {
513 qemu_log("qemu: fatal: ");
514 qemu_log_vprintf(fmt, ap2);
515 qemu_log("\n");
516 log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
517 qemu_log_flush();
518 qemu_log_close();
519 }
520 va_end(ap2);
521 va_end(ap);
522 #if defined(CONFIG_USER_ONLY)
523 {
524 struct sigaction act;
525 sigfillset(&act.sa_mask);
526 act.sa_handler = SIG_DFL;
527 sigaction(SIGABRT, &act, NULL);
528 }
529 #endif
530 abort();
531 }
532
533 CPUArchState *cpu_copy(CPUArchState *env)
534 {
535 CPUArchState *new_env = cpu_init(env->cpu_model_str);
536 CPUArchState *next_cpu = new_env->next_cpu;
537 #if defined(TARGET_HAS_ICE)
538 CPUBreakpoint *bp;
539 CPUWatchpoint *wp;
540 #endif
541
542 memcpy(new_env, env, sizeof(CPUArchState));
543
544 /* Preserve chaining. */
545 new_env->next_cpu = next_cpu;
546
547 /* Clone all break/watchpoints.
548 Note: Once we support ptrace with hw-debug register access, make sure
549 BP_CPU break/watchpoints are handled correctly on clone. */
550 QTAILQ_INIT(&env->breakpoints);
551 QTAILQ_INIT(&env->watchpoints);
552 #if defined(TARGET_HAS_ICE)
553 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
554 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
555 }
556 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
557 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
558 wp->flags, NULL);
559 }
560 #endif
561
562 return new_env;
563 }
564
565 #if !defined(CONFIG_USER_ONLY)
566 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
567 uintptr_t length)
568 {
569 uintptr_t start1;
570
571 /* we modify the TLB cache so that the dirty bit will be set again
572 when accessing the range */
573 start1 = (uintptr_t)qemu_safe_ram_ptr(start);
574 /* Check that we don't span multiple blocks - this breaks the
575 address comparisons below. */
576 if ((uintptr_t)qemu_safe_ram_ptr(end - 1) - start1
577 != (end - 1) - start) {
578 abort();
579 }
580 cpu_tlb_reset_dirty_all(start1, length);
581
582 }
583
584 /* Note: start and end must be within the same ram block. */
585 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
586 int dirty_flags)
587 {
588 uintptr_t length;
589
590 start &= TARGET_PAGE_MASK;
591 end = TARGET_PAGE_ALIGN(end);
592
593 length = end - start;
594 if (length == 0)
595 return;
596 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
597
598 if (tcg_enabled()) {
599 tlb_reset_dirty_range_all(start, end, length);
600 }
601 }
602
603 static int cpu_physical_memory_set_dirty_tracking(int enable)
604 {
605 int ret = 0;
606 in_migration = enable;
607 return ret;
608 }
609
610 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
611 MemoryRegionSection *section,
612 target_ulong vaddr,
613 hwaddr paddr,
614 int prot,
615 target_ulong *address)
616 {
617 hwaddr iotlb;
618 CPUWatchpoint *wp;
619
620 if (memory_region_is_ram(section->mr)) {
621 /* Normal RAM. */
622 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
623 + memory_region_section_addr(section, paddr);
624 if (!section->readonly) {
625 iotlb |= phys_section_notdirty;
626 } else {
627 iotlb |= phys_section_rom;
628 }
629 } else {
630 /* IO handlers are currently passed a physical address.
631 It would be nice to pass an offset from the base address
632 of that region. This would avoid having to special case RAM,
633 and avoid full address decoding in every device.
634 We can't use the high bits of pd for this because
635 IO_MEM_ROMD uses these as a ram address. */
636 iotlb = section - phys_sections;
637 iotlb += memory_region_section_addr(section, paddr);
638 }
639
640 /* Make accesses to pages with watchpoints go via the
641 watchpoint trap routines. */
642 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
643 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
644 /* Avoid trapping reads of pages with a write breakpoint. */
645 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
646 iotlb = phys_section_watch + paddr;
647 *address |= TLB_MMIO;
648 break;
649 }
650 }
651 }
652
653 return iotlb;
654 }
655 #endif /* defined(CONFIG_USER_ONLY) */
656
657 #if !defined(CONFIG_USER_ONLY)
658
659 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
660 typedef struct subpage_t {
661 MemoryRegion iomem;
662 hwaddr base;
663 uint16_t sub_section[TARGET_PAGE_SIZE];
664 } subpage_t;
665
666 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
667 uint16_t section);
668 static subpage_t *subpage_init(hwaddr base);
669 static void destroy_page_desc(uint16_t section_index)
670 {
671 MemoryRegionSection *section = &phys_sections[section_index];
672 MemoryRegion *mr = section->mr;
673
674 if (mr->subpage) {
675 subpage_t *subpage = container_of(mr, subpage_t, iomem);
676 memory_region_destroy(&subpage->iomem);
677 g_free(subpage);
678 }
679 }
680
681 static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
682 {
683 unsigned i;
684 PhysPageEntry *p;
685
686 if (lp->ptr == PHYS_MAP_NODE_NIL) {
687 return;
688 }
689
690 p = phys_map_nodes[lp->ptr];
691 for (i = 0; i < L2_SIZE; ++i) {
692 if (!p[i].is_leaf) {
693 destroy_l2_mapping(&p[i], level - 1);
694 } else {
695 destroy_page_desc(p[i].ptr);
696 }
697 }
698 lp->is_leaf = 0;
699 lp->ptr = PHYS_MAP_NODE_NIL;
700 }
701
702 static void destroy_all_mappings(AddressSpaceDispatch *d)
703 {
704 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
705 phys_map_nodes_reset();
706 }
707
708 static uint16_t phys_section_add(MemoryRegionSection *section)
709 {
710 if (phys_sections_nb == phys_sections_nb_alloc) {
711 phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
712 phys_sections = g_renew(MemoryRegionSection, phys_sections,
713 phys_sections_nb_alloc);
714 }
715 phys_sections[phys_sections_nb] = *section;
716 return phys_sections_nb++;
717 }
718
719 static void phys_sections_clear(void)
720 {
721 phys_sections_nb = 0;
722 }
723
724 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
725 {
726 subpage_t *subpage;
727 hwaddr base = section->offset_within_address_space
728 & TARGET_PAGE_MASK;
729 MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
730 MemoryRegionSection subsection = {
731 .offset_within_address_space = base,
732 .size = TARGET_PAGE_SIZE,
733 };
734 hwaddr start, end;
735
736 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
737
738 if (!(existing->mr->subpage)) {
739 subpage = subpage_init(base);
740 subsection.mr = &subpage->iomem;
741 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
742 phys_section_add(&subsection));
743 } else {
744 subpage = container_of(existing->mr, subpage_t, iomem);
745 }
746 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
747 end = start + section->size - 1;
748 subpage_register(subpage, start, end, phys_section_add(section));
749 }
750
751
752 static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
753 {
754 hwaddr start_addr = section->offset_within_address_space;
755 ram_addr_t size = section->size;
756 hwaddr addr;
757 uint16_t section_index = phys_section_add(section);
758
759 assert(size);
760
761 addr = start_addr;
762 phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
763 section_index);
764 }
765
766 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
767 {
768 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
769 MemoryRegionSection now = *section, remain = *section;
770
771 if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
772 || (now.size < TARGET_PAGE_SIZE)) {
773 now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
774 - now.offset_within_address_space,
775 now.size);
776 register_subpage(d, &now);
777 remain.size -= now.size;
778 remain.offset_within_address_space += now.size;
779 remain.offset_within_region += now.size;
780 }
781 while (remain.size >= TARGET_PAGE_SIZE) {
782 now = remain;
783 if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
784 now.size = TARGET_PAGE_SIZE;
785 register_subpage(d, &now);
786 } else {
787 now.size &= TARGET_PAGE_MASK;
788 register_multipage(d, &now);
789 }
790 remain.size -= now.size;
791 remain.offset_within_address_space += now.size;
792 remain.offset_within_region += now.size;
793 }
794 now = remain;
795 if (now.size) {
796 register_subpage(d, &now);
797 }
798 }
799
800 void qemu_flush_coalesced_mmio_buffer(void)
801 {
802 if (kvm_enabled())
803 kvm_flush_coalesced_mmio_buffer();
804 }
805
806 void qemu_mutex_lock_ramlist(void)
807 {
808 qemu_mutex_lock(&ram_list.mutex);
809 }
810
811 void qemu_mutex_unlock_ramlist(void)
812 {
813 qemu_mutex_unlock(&ram_list.mutex);
814 }
815
816 #if defined(__linux__) && !defined(TARGET_S390X)
817
818 #include <sys/vfs.h>
819
820 #define HUGETLBFS_MAGIC 0x958458f6
821
822 static long gethugepagesize(const char *path)
823 {
824 struct statfs fs;
825 int ret;
826
827 do {
828 ret = statfs(path, &fs);
829 } while (ret != 0 && errno == EINTR);
830
831 if (ret != 0) {
832 perror(path);
833 return 0;
834 }
835
836 if (fs.f_type != HUGETLBFS_MAGIC)
837 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
838
839 return fs.f_bsize;
840 }
841
842 static void *file_ram_alloc(RAMBlock *block,
843 ram_addr_t memory,
844 const char *path)
845 {
846 char *filename;
847 void *area;
848 int fd;
849 #ifdef MAP_POPULATE
850 int flags;
851 #endif
852 unsigned long hpagesize;
853
854 hpagesize = gethugepagesize(path);
855 if (!hpagesize) {
856 return NULL;
857 }
858
859 if (memory < hpagesize) {
860 return NULL;
861 }
862
863 if (kvm_enabled() && !kvm_has_sync_mmu()) {
864 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
865 return NULL;
866 }
867
868 filename = g_strdup_printf("%s/qemu_back_mem.XXXXXX", path);
869
870 fd = mkstemp(filename);
871 if (fd < 0) {
872 perror("unable to create backing store for hugepages");
873 g_free(filename);
874 return NULL;
875 }
876 unlink(filename);
877 g_free(filename);
878
879 memory = (memory+hpagesize-1) & ~(hpagesize-1);
880
881 /*
882 * ftruncate is not supported by hugetlbfs in older
883 * hosts, so don't bother bailing out on errors.
884 * If anything goes wrong with it under other filesystems,
885 * mmap will fail.
886 */
887 if (ftruncate(fd, memory))
888 perror("ftruncate");
889
890 #ifdef MAP_POPULATE
891 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
892 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
893 * to sidestep this quirk.
894 */
895 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
896 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
897 #else
898 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
899 #endif
900 if (area == MAP_FAILED) {
901 perror("file_ram_alloc: can't mmap RAM pages");
902 close(fd);
903 return (NULL);
904 }
905 block->fd = fd;
906 return area;
907 }
908 #endif
909
910 static ram_addr_t find_ram_offset(ram_addr_t size)
911 {
912 RAMBlock *block, *next_block;
913 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
914
915 if (QTAILQ_EMPTY(&ram_list.blocks))
916 return 0;
917
918 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
919 ram_addr_t end, next = RAM_ADDR_MAX;
920
921 end = block->offset + block->length;
922
923 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
924 if (next_block->offset >= end) {
925 next = MIN(next, next_block->offset);
926 }
927 }
928 if (next - end >= size && next - end < mingap) {
929 offset = end;
930 mingap = next - end;
931 }
932 }
933
934 if (offset == RAM_ADDR_MAX) {
935 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
936 (uint64_t)size);
937 abort();
938 }
939
940 return offset;
941 }
942
943 ram_addr_t last_ram_offset(void)
944 {
945 RAMBlock *block;
946 ram_addr_t last = 0;
947
948 QTAILQ_FOREACH(block, &ram_list.blocks, next)
949 last = MAX(last, block->offset + block->length);
950
951 return last;
952 }
953
954 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
955 {
956 int ret;
957 QemuOpts *machine_opts;
958
959 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
960 machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
961 if (machine_opts &&
962 !qemu_opt_get_bool(machine_opts, "dump-guest-core", true)) {
963 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
964 if (ret) {
965 perror("qemu_madvise");
966 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
967 "but dump_guest_core=off specified\n");
968 }
969 }
970 }
971
972 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
973 {
974 RAMBlock *new_block, *block;
975
976 new_block = NULL;
977 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
978 if (block->offset == addr) {
979 new_block = block;
980 break;
981 }
982 }
983 assert(new_block);
984 assert(!new_block->idstr[0]);
985
986 if (dev) {
987 char *id = qdev_get_dev_path(dev);
988 if (id) {
989 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
990 g_free(id);
991 }
992 }
993 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
994
995 /* This assumes the iothread lock is taken here too. */
996 qemu_mutex_lock_ramlist();
997 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
998 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
999 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1000 new_block->idstr);
1001 abort();
1002 }
1003 }
1004 qemu_mutex_unlock_ramlist();
1005 }
1006
1007 static int memory_try_enable_merging(void *addr, size_t len)
1008 {
1009 QemuOpts *opts;
1010
1011 opts = qemu_opts_find(qemu_find_opts("machine"), 0);
1012 if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
1013 /* disabled by the user */
1014 return 0;
1015 }
1016
1017 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1018 }
1019
1020 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1021 MemoryRegion *mr)
1022 {
1023 RAMBlock *block, *new_block;
1024
1025 size = TARGET_PAGE_ALIGN(size);
1026 new_block = g_malloc0(sizeof(*new_block));
1027
1028 /* This assumes the iothread lock is taken here too. */
1029 qemu_mutex_lock_ramlist();
1030 new_block->mr = mr;
1031 new_block->offset = find_ram_offset(size);
1032 if (host) {
1033 new_block->host = host;
1034 new_block->flags |= RAM_PREALLOC_MASK;
1035 } else {
1036 if (mem_path) {
1037 #if defined (__linux__) && !defined(TARGET_S390X)
1038 new_block->host = file_ram_alloc(new_block, size, mem_path);
1039 if (!new_block->host) {
1040 new_block->host = qemu_vmalloc(size);
1041 memory_try_enable_merging(new_block->host, size);
1042 }
1043 #else
1044 fprintf(stderr, "-mem-path option unsupported\n");
1045 exit(1);
1046 #endif
1047 } else {
1048 if (xen_enabled()) {
1049 xen_ram_alloc(new_block->offset, size, mr);
1050 } else if (kvm_enabled()) {
1051 /* some s390/kvm configurations have special constraints */
1052 new_block->host = kvm_vmalloc(size);
1053 } else {
1054 new_block->host = qemu_vmalloc(size);
1055 }
1056 memory_try_enable_merging(new_block->host, size);
1057 }
1058 }
1059 new_block->length = size;
1060
1061 /* Keep the list sorted from biggest to smallest block. */
1062 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1063 if (block->length < new_block->length) {
1064 break;
1065 }
1066 }
1067 if (block) {
1068 QTAILQ_INSERT_BEFORE(block, new_block, next);
1069 } else {
1070 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1071 }
1072 ram_list.mru_block = NULL;
1073
1074 ram_list.version++;
1075 qemu_mutex_unlock_ramlist();
1076
1077 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty,
1078 last_ram_offset() >> TARGET_PAGE_BITS);
1079 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1080 0, size >> TARGET_PAGE_BITS);
1081 cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
1082
1083 qemu_ram_setup_dump(new_block->host, size);
1084 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1085
1086 if (kvm_enabled())
1087 kvm_setup_guest_memory(new_block->host, size);
1088
1089 return new_block->offset;
1090 }
1091
1092 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1093 {
1094 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1095 }
1096
1097 void qemu_ram_free_from_ptr(ram_addr_t addr)
1098 {
1099 RAMBlock *block;
1100
1101 /* This assumes the iothread lock is taken here too. */
1102 qemu_mutex_lock_ramlist();
1103 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1104 if (addr == block->offset) {
1105 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1106 ram_list.mru_block = NULL;
1107 ram_list.version++;
1108 g_free(block);
1109 break;
1110 }
1111 }
1112 qemu_mutex_unlock_ramlist();
1113 }
1114
1115 void qemu_ram_free(ram_addr_t addr)
1116 {
1117 RAMBlock *block;
1118
1119 /* This assumes the iothread lock is taken here too. */
1120 qemu_mutex_lock_ramlist();
1121 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1122 if (addr == block->offset) {
1123 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1124 ram_list.mru_block = NULL;
1125 ram_list.version++;
1126 if (block->flags & RAM_PREALLOC_MASK) {
1127 ;
1128 } else if (mem_path) {
1129 #if defined (__linux__) && !defined(TARGET_S390X)
1130 if (block->fd) {
1131 munmap(block->host, block->length);
1132 close(block->fd);
1133 } else {
1134 qemu_vfree(block->host);
1135 }
1136 #else
1137 abort();
1138 #endif
1139 } else {
1140 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1141 munmap(block->host, block->length);
1142 #else
1143 if (xen_enabled()) {
1144 xen_invalidate_map_cache_entry(block->host);
1145 } else {
1146 qemu_vfree(block->host);
1147 }
1148 #endif
1149 }
1150 g_free(block);
1151 break;
1152 }
1153 }
1154 qemu_mutex_unlock_ramlist();
1155
1156 }
1157
1158 #ifndef _WIN32
1159 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1160 {
1161 RAMBlock *block;
1162 ram_addr_t offset;
1163 int flags;
1164 void *area, *vaddr;
1165
1166 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1167 offset = addr - block->offset;
1168 if (offset < block->length) {
1169 vaddr = block->host + offset;
1170 if (block->flags & RAM_PREALLOC_MASK) {
1171 ;
1172 } else {
1173 flags = MAP_FIXED;
1174 munmap(vaddr, length);
1175 if (mem_path) {
1176 #if defined(__linux__) && !defined(TARGET_S390X)
1177 if (block->fd) {
1178 #ifdef MAP_POPULATE
1179 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1180 MAP_PRIVATE;
1181 #else
1182 flags |= MAP_PRIVATE;
1183 #endif
1184 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1185 flags, block->fd, offset);
1186 } else {
1187 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1188 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1189 flags, -1, 0);
1190 }
1191 #else
1192 abort();
1193 #endif
1194 } else {
1195 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
1196 flags |= MAP_SHARED | MAP_ANONYMOUS;
1197 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
1198 flags, -1, 0);
1199 #else
1200 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1201 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1202 flags, -1, 0);
1203 #endif
1204 }
1205 if (area != vaddr) {
1206 fprintf(stderr, "Could not remap addr: "
1207 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1208 length, addr);
1209 exit(1);
1210 }
1211 memory_try_enable_merging(vaddr, length);
1212 qemu_ram_setup_dump(vaddr, length);
1213 }
1214 return;
1215 }
1216 }
1217 }
1218 #endif /* !_WIN32 */
1219
1220 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1221 With the exception of the softmmu code in this file, this should
1222 only be used for local memory (e.g. video ram) that the device owns,
1223 and knows it isn't going to access beyond the end of the block.
1224
1225 It should not be used for general purpose DMA.
1226 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1227 */
1228 void *qemu_get_ram_ptr(ram_addr_t addr)
1229 {
1230 RAMBlock *block;
1231
1232 /* The list is protected by the iothread lock here. */
1233 block = ram_list.mru_block;
1234 if (block && addr - block->offset < block->length) {
1235 goto found;
1236 }
1237 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1238 if (addr - block->offset < block->length) {
1239 goto found;
1240 }
1241 }
1242
1243 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1244 abort();
1245
1246 found:
1247 ram_list.mru_block = block;
1248 if (xen_enabled()) {
1249 /* We need to check if the requested address is in the RAM
1250 * because we don't want to map the entire memory in QEMU.
1251 * In that case just map until the end of the page.
1252 */
1253 if (block->offset == 0) {
1254 return xen_map_cache(addr, 0, 0);
1255 } else if (block->host == NULL) {
1256 block->host =
1257 xen_map_cache(block->offset, block->length, 1);
1258 }
1259 }
1260 return block->host + (addr - block->offset);
1261 }
1262
1263 /* Return a host pointer to ram allocated with qemu_ram_alloc. Same as
1264 * qemu_get_ram_ptr but do not touch ram_list.mru_block.
1265 *
1266 * ??? Is this still necessary?
1267 */
1268 static void *qemu_safe_ram_ptr(ram_addr_t addr)
1269 {
1270 RAMBlock *block;
1271
1272 /* The list is protected by the iothread lock here. */
1273 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1274 if (addr - block->offset < block->length) {
1275 if (xen_enabled()) {
1276 /* We need to check if the requested address is in the RAM
1277 * because we don't want to map the entire memory in QEMU.
1278 * In that case just map until the end of the page.
1279 */
1280 if (block->offset == 0) {
1281 return xen_map_cache(addr, 0, 0);
1282 } else if (block->host == NULL) {
1283 block->host =
1284 xen_map_cache(block->offset, block->length, 1);
1285 }
1286 }
1287 return block->host + (addr - block->offset);
1288 }
1289 }
1290
1291 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1292 abort();
1293
1294 return NULL;
1295 }
1296
1297 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1298 * but takes a size argument */
1299 static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
1300 {
1301 if (*size == 0) {
1302 return NULL;
1303 }
1304 if (xen_enabled()) {
1305 return xen_map_cache(addr, *size, 1);
1306 } else {
1307 RAMBlock *block;
1308
1309 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1310 if (addr - block->offset < block->length) {
1311 if (addr - block->offset + *size > block->length)
1312 *size = block->length - addr + block->offset;
1313 return block->host + (addr - block->offset);
1314 }
1315 }
1316
1317 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1318 abort();
1319 }
1320 }
1321
1322 void qemu_put_ram_ptr(void *addr)
1323 {
1324 trace_qemu_put_ram_ptr(addr);
1325 }
1326
1327 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1328 {
1329 RAMBlock *block;
1330 uint8_t *host = ptr;
1331
1332 if (xen_enabled()) {
1333 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1334 return 0;
1335 }
1336
1337 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1338 /* This case append when the block is not mapped. */
1339 if (block->host == NULL) {
1340 continue;
1341 }
1342 if (host - block->host < block->length) {
1343 *ram_addr = block->offset + (host - block->host);
1344 return 0;
1345 }
1346 }
1347
1348 return -1;
1349 }
1350
1351 /* Some of the softmmu routines need to translate from a host pointer
1352 (typically a TLB entry) back to a ram offset. */
1353 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
1354 {
1355 ram_addr_t ram_addr;
1356
1357 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
1358 fprintf(stderr, "Bad ram pointer %p\n", ptr);
1359 abort();
1360 }
1361 return ram_addr;
1362 }
1363
1364 static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
1365 unsigned size)
1366 {
1367 #ifdef DEBUG_UNASSIGNED
1368 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
1369 #endif
1370 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1371 cpu_unassigned_access(cpu_single_env, addr, 0, 0, 0, size);
1372 #endif
1373 return 0;
1374 }
1375
1376 static void unassigned_mem_write(void *opaque, hwaddr addr,
1377 uint64_t val, unsigned size)
1378 {
1379 #ifdef DEBUG_UNASSIGNED
1380 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%"PRIx64"\n", addr, val);
1381 #endif
1382 #if defined(TARGET_ALPHA) || defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
1383 cpu_unassigned_access(cpu_single_env, addr, 1, 0, 0, size);
1384 #endif
1385 }
1386
1387 static const MemoryRegionOps unassigned_mem_ops = {
1388 .read = unassigned_mem_read,
1389 .write = unassigned_mem_write,
1390 .endianness = DEVICE_NATIVE_ENDIAN,
1391 };
1392
1393 static uint64_t error_mem_read(void *opaque, hwaddr addr,
1394 unsigned size)
1395 {
1396 abort();
1397 }
1398
1399 static void error_mem_write(void *opaque, hwaddr addr,
1400 uint64_t value, unsigned size)
1401 {
1402 abort();
1403 }
1404
1405 static const MemoryRegionOps error_mem_ops = {
1406 .read = error_mem_read,
1407 .write = error_mem_write,
1408 .endianness = DEVICE_NATIVE_ENDIAN,
1409 };
1410
1411 static const MemoryRegionOps rom_mem_ops = {
1412 .read = error_mem_read,
1413 .write = unassigned_mem_write,
1414 .endianness = DEVICE_NATIVE_ENDIAN,
1415 };
1416
1417 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1418 uint64_t val, unsigned size)
1419 {
1420 int dirty_flags;
1421 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1422 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
1423 #if !defined(CONFIG_USER_ONLY)
1424 tb_invalidate_phys_page_fast(ram_addr, size);
1425 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
1426 #endif
1427 }
1428 switch (size) {
1429 case 1:
1430 stb_p(qemu_get_ram_ptr(ram_addr), val);
1431 break;
1432 case 2:
1433 stw_p(qemu_get_ram_ptr(ram_addr), val);
1434 break;
1435 case 4:
1436 stl_p(qemu_get_ram_ptr(ram_addr), val);
1437 break;
1438 default:
1439 abort();
1440 }
1441 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
1442 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
1443 /* we remove the notdirty callback only if the code has been
1444 flushed */
1445 if (dirty_flags == 0xff)
1446 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
1447 }
1448
1449 static const MemoryRegionOps notdirty_mem_ops = {
1450 .read = error_mem_read,
1451 .write = notdirty_mem_write,
1452 .endianness = DEVICE_NATIVE_ENDIAN,
1453 };
1454
1455 /* Generate a debug exception if a watchpoint has been hit. */
1456 static void check_watchpoint(int offset, int len_mask, int flags)
1457 {
1458 CPUArchState *env = cpu_single_env;
1459 target_ulong pc, cs_base;
1460 target_ulong vaddr;
1461 CPUWatchpoint *wp;
1462 int cpu_flags;
1463
1464 if (env->watchpoint_hit) {
1465 /* We re-entered the check after replacing the TB. Now raise
1466 * the debug interrupt so that is will trigger after the
1467 * current instruction. */
1468 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
1469 return;
1470 }
1471 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1472 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1473 if ((vaddr == (wp->vaddr & len_mask) ||
1474 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1475 wp->flags |= BP_WATCHPOINT_HIT;
1476 if (!env->watchpoint_hit) {
1477 env->watchpoint_hit = wp;
1478 tb_check_watchpoint(env);
1479 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1480 env->exception_index = EXCP_DEBUG;
1481 cpu_loop_exit(env);
1482 } else {
1483 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1484 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1485 cpu_resume_from_signal(env, NULL);
1486 }
1487 }
1488 } else {
1489 wp->flags &= ~BP_WATCHPOINT_HIT;
1490 }
1491 }
1492 }
1493
1494 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1495 so these check for a hit then pass through to the normal out-of-line
1496 phys routines. */
1497 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1498 unsigned size)
1499 {
1500 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1501 switch (size) {
1502 case 1: return ldub_phys(addr);
1503 case 2: return lduw_phys(addr);
1504 case 4: return ldl_phys(addr);
1505 default: abort();
1506 }
1507 }
1508
1509 static void watch_mem_write(void *opaque, hwaddr addr,
1510 uint64_t val, unsigned size)
1511 {
1512 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1513 switch (size) {
1514 case 1:
1515 stb_phys(addr, val);
1516 break;
1517 case 2:
1518 stw_phys(addr, val);
1519 break;
1520 case 4:
1521 stl_phys(addr, val);
1522 break;
1523 default: abort();
1524 }
1525 }
1526
1527 static const MemoryRegionOps watch_mem_ops = {
1528 .read = watch_mem_read,
1529 .write = watch_mem_write,
1530 .endianness = DEVICE_NATIVE_ENDIAN,
1531 };
1532
1533 static uint64_t subpage_read(void *opaque, hwaddr addr,
1534 unsigned len)
1535 {
1536 subpage_t *mmio = opaque;
1537 unsigned int idx = SUBPAGE_IDX(addr);
1538 MemoryRegionSection *section;
1539 #if defined(DEBUG_SUBPAGE)
1540 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
1541 mmio, len, addr, idx);
1542 #endif
1543
1544 section = &phys_sections[mmio->sub_section[idx]];
1545 addr += mmio->base;
1546 addr -= section->offset_within_address_space;
1547 addr += section->offset_within_region;
1548 return io_mem_read(section->mr, addr, len);
1549 }
1550
1551 static void subpage_write(void *opaque, hwaddr addr,
1552 uint64_t value, unsigned len)
1553 {
1554 subpage_t *mmio = opaque;
1555 unsigned int idx = SUBPAGE_IDX(addr);
1556 MemoryRegionSection *section;
1557 #if defined(DEBUG_SUBPAGE)
1558 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
1559 " idx %d value %"PRIx64"\n",
1560 __func__, mmio, len, addr, idx, value);
1561 #endif
1562
1563 section = &phys_sections[mmio->sub_section[idx]];
1564 addr += mmio->base;
1565 addr -= section->offset_within_address_space;
1566 addr += section->offset_within_region;
1567 io_mem_write(section->mr, addr, value, len);
1568 }
1569
1570 static const MemoryRegionOps subpage_ops = {
1571 .read = subpage_read,
1572 .write = subpage_write,
1573 .endianness = DEVICE_NATIVE_ENDIAN,
1574 };
1575
1576 static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
1577 unsigned size)
1578 {
1579 ram_addr_t raddr = addr;
1580 void *ptr = qemu_get_ram_ptr(raddr);
1581 switch (size) {
1582 case 1: return ldub_p(ptr);
1583 case 2: return lduw_p(ptr);
1584 case 4: return ldl_p(ptr);
1585 default: abort();
1586 }
1587 }
1588
1589 static void subpage_ram_write(void *opaque, hwaddr addr,
1590 uint64_t value, unsigned size)
1591 {
1592 ram_addr_t raddr = addr;
1593 void *ptr = qemu_get_ram_ptr(raddr);
1594 switch (size) {
1595 case 1: return stb_p(ptr, value);
1596 case 2: return stw_p(ptr, value);
1597 case 4: return stl_p(ptr, value);
1598 default: abort();
1599 }
1600 }
1601
1602 static const MemoryRegionOps subpage_ram_ops = {
1603 .read = subpage_ram_read,
1604 .write = subpage_ram_write,
1605 .endianness = DEVICE_NATIVE_ENDIAN,
1606 };
1607
1608 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1609 uint16_t section)
1610 {
1611 int idx, eidx;
1612
1613 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1614 return -1;
1615 idx = SUBPAGE_IDX(start);
1616 eidx = SUBPAGE_IDX(end);
1617 #if defined(DEBUG_SUBPAGE)
1618 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
1619 mmio, start, end, idx, eidx, memory);
1620 #endif
1621 if (memory_region_is_ram(phys_sections[section].mr)) {
1622 MemoryRegionSection new_section = phys_sections[section];
1623 new_section.mr = &io_mem_subpage_ram;
1624 section = phys_section_add(&new_section);
1625 }
1626 for (; idx <= eidx; idx++) {
1627 mmio->sub_section[idx] = section;
1628 }
1629
1630 return 0;
1631 }
1632
1633 static subpage_t *subpage_init(hwaddr base)
1634 {
1635 subpage_t *mmio;
1636
1637 mmio = g_malloc0(sizeof(subpage_t));
1638
1639 mmio->base = base;
1640 memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
1641 "subpage", TARGET_PAGE_SIZE);
1642 mmio->iomem.subpage = true;
1643 #if defined(DEBUG_SUBPAGE)
1644 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
1645 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
1646 #endif
1647 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
1648
1649 return mmio;
1650 }
1651
1652 static uint16_t dummy_section(MemoryRegion *mr)
1653 {
1654 MemoryRegionSection section = {
1655 .mr = mr,
1656 .offset_within_address_space = 0,
1657 .offset_within_region = 0,
1658 .size = UINT64_MAX,
1659 };
1660
1661 return phys_section_add(&section);
1662 }
1663
1664 MemoryRegion *iotlb_to_region(hwaddr index)
1665 {
1666 return phys_sections[index & ~TARGET_PAGE_MASK].mr;
1667 }
1668
1669 static void io_mem_init(void)
1670 {
1671 memory_region_init_io(&io_mem_ram, &error_mem_ops, NULL, "ram", UINT64_MAX);
1672 memory_region_init_io(&io_mem_rom, &rom_mem_ops, NULL, "rom", UINT64_MAX);
1673 memory_region_init_io(&io_mem_unassigned, &unassigned_mem_ops, NULL,
1674 "unassigned", UINT64_MAX);
1675 memory_region_init_io(&io_mem_notdirty, &notdirty_mem_ops, NULL,
1676 "notdirty", UINT64_MAX);
1677 memory_region_init_io(&io_mem_subpage_ram, &subpage_ram_ops, NULL,
1678 "subpage-ram", UINT64_MAX);
1679 memory_region_init_io(&io_mem_watch, &watch_mem_ops, NULL,
1680 "watch", UINT64_MAX);
1681 }
1682
1683 static void mem_begin(MemoryListener *listener)
1684 {
1685 AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
1686
1687 destroy_all_mappings(d);
1688 d->phys_map.ptr = PHYS_MAP_NODE_NIL;
1689 }
1690
1691 static void core_begin(MemoryListener *listener)
1692 {
1693 phys_sections_clear();
1694 phys_section_unassigned = dummy_section(&io_mem_unassigned);
1695 phys_section_notdirty = dummy_section(&io_mem_notdirty);
1696 phys_section_rom = dummy_section(&io_mem_rom);
1697 phys_section_watch = dummy_section(&io_mem_watch);
1698 }
1699
1700 static void tcg_commit(MemoryListener *listener)
1701 {
1702 CPUArchState *env;
1703
1704 /* since each CPU stores ram addresses in its TLB cache, we must
1705 reset the modified entries */
1706 /* XXX: slow ! */
1707 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1708 tlb_flush(env, 1);
1709 }
1710 }
1711
1712 static void core_log_global_start(MemoryListener *listener)
1713 {
1714 cpu_physical_memory_set_dirty_tracking(1);
1715 }
1716
1717 static void core_log_global_stop(MemoryListener *listener)
1718 {
1719 cpu_physical_memory_set_dirty_tracking(0);
1720 }
1721
1722 static void io_region_add(MemoryListener *listener,
1723 MemoryRegionSection *section)
1724 {
1725 MemoryRegionIORange *mrio = g_new(MemoryRegionIORange, 1);
1726
1727 mrio->mr = section->mr;
1728 mrio->offset = section->offset_within_region;
1729 iorange_init(&mrio->iorange, &memory_region_iorange_ops,
1730 section->offset_within_address_space, section->size);
1731 ioport_register(&mrio->iorange);
1732 }
1733
1734 static void io_region_del(MemoryListener *listener,
1735 MemoryRegionSection *section)
1736 {
1737 isa_unassign_ioport(section->offset_within_address_space, section->size);
1738 }
1739
1740 static MemoryListener core_memory_listener = {
1741 .begin = core_begin,
1742 .log_global_start = core_log_global_start,
1743 .log_global_stop = core_log_global_stop,
1744 .priority = 1,
1745 };
1746
1747 static MemoryListener io_memory_listener = {
1748 .region_add = io_region_add,
1749 .region_del = io_region_del,
1750 .priority = 0,
1751 };
1752
1753 static MemoryListener tcg_memory_listener = {
1754 .commit = tcg_commit,
1755 };
1756
1757 void address_space_init_dispatch(AddressSpace *as)
1758 {
1759 AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
1760
1761 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
1762 d->listener = (MemoryListener) {
1763 .begin = mem_begin,
1764 .region_add = mem_add,
1765 .region_nop = mem_add,
1766 .priority = 0,
1767 };
1768 as->dispatch = d;
1769 memory_listener_register(&d->listener, as);
1770 }
1771
1772 void address_space_destroy_dispatch(AddressSpace *as)
1773 {
1774 AddressSpaceDispatch *d = as->dispatch;
1775
1776 memory_listener_unregister(&d->listener);
1777 destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
1778 g_free(d);
1779 as->dispatch = NULL;
1780 }
1781
1782 static void memory_map_init(void)
1783 {
1784 system_memory = g_malloc(sizeof(*system_memory));
1785 memory_region_init(system_memory, "system", INT64_MAX);
1786 address_space_init(&address_space_memory, system_memory);
1787 address_space_memory.name = "memory";
1788
1789 system_io = g_malloc(sizeof(*system_io));
1790 memory_region_init(system_io, "io", 65536);
1791 address_space_init(&address_space_io, system_io);
1792 address_space_io.name = "I/O";
1793
1794 memory_listener_register(&core_memory_listener, &address_space_memory);
1795 memory_listener_register(&io_memory_listener, &address_space_io);
1796 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1797
1798 dma_context_init(&dma_context_memory, &address_space_memory,
1799 NULL, NULL, NULL);
1800 }
1801
1802 MemoryRegion *get_system_memory(void)
1803 {
1804 return system_memory;
1805 }
1806
1807 MemoryRegion *get_system_io(void)
1808 {
1809 return system_io;
1810 }
1811
1812 #endif /* !defined(CONFIG_USER_ONLY) */
1813
1814 /* physical memory access (slow version, mainly for debug) */
1815 #if defined(CONFIG_USER_ONLY)
1816 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
1817 uint8_t *buf, int len, int is_write)
1818 {
1819 int l, flags;
1820 target_ulong page;
1821 void * p;
1822
1823 while (len > 0) {
1824 page = addr & TARGET_PAGE_MASK;
1825 l = (page + TARGET_PAGE_SIZE) - addr;
1826 if (l > len)
1827 l = len;
1828 flags = page_get_flags(page);
1829 if (!(flags & PAGE_VALID))
1830 return -1;
1831 if (is_write) {
1832 if (!(flags & PAGE_WRITE))
1833 return -1;
1834 /* XXX: this code should not depend on lock_user */
1835 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1836 return -1;
1837 memcpy(p, buf, l);
1838 unlock_user(p, addr, l);
1839 } else {
1840 if (!(flags & PAGE_READ))
1841 return -1;
1842 /* XXX: this code should not depend on lock_user */
1843 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1844 return -1;
1845 memcpy(buf, p, l);
1846 unlock_user(p, addr, 0);
1847 }
1848 len -= l;
1849 buf += l;
1850 addr += l;
1851 }
1852 return 0;
1853 }
1854
1855 #else
1856
1857 static void invalidate_and_set_dirty(hwaddr addr,
1858 hwaddr length)
1859 {
1860 if (!cpu_physical_memory_is_dirty(addr)) {
1861 /* invalidate code */
1862 tb_invalidate_phys_page_range(addr, addr + length, 0);
1863 /* set dirty bit */
1864 cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
1865 }
1866 xen_modified_memory(addr, length);
1867 }
1868
1869 void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1870 int len, bool is_write)
1871 {
1872 AddressSpaceDispatch *d = as->dispatch;
1873 int l;
1874 uint8_t *ptr;
1875 uint32_t val;
1876 hwaddr page;
1877 MemoryRegionSection *section;
1878
1879 while (len > 0) {
1880 page = addr & TARGET_PAGE_MASK;
1881 l = (page + TARGET_PAGE_SIZE) - addr;
1882 if (l > len)
1883 l = len;
1884 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1885
1886 if (is_write) {
1887 if (!memory_region_is_ram(section->mr)) {
1888 hwaddr addr1;
1889 addr1 = memory_region_section_addr(section, addr);
1890 /* XXX: could force cpu_single_env to NULL to avoid
1891 potential bugs */
1892 if (l >= 4 && ((addr1 & 3) == 0)) {
1893 /* 32 bit write access */
1894 val = ldl_p(buf);
1895 io_mem_write(section->mr, addr1, val, 4);
1896 l = 4;
1897 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1898 /* 16 bit write access */
1899 val = lduw_p(buf);
1900 io_mem_write(section->mr, addr1, val, 2);
1901 l = 2;
1902 } else {
1903 /* 8 bit write access */
1904 val = ldub_p(buf);
1905 io_mem_write(section->mr, addr1, val, 1);
1906 l = 1;
1907 }
1908 } else if (!section->readonly) {
1909 ram_addr_t addr1;
1910 addr1 = memory_region_get_ram_addr(section->mr)
1911 + memory_region_section_addr(section, addr);
1912 /* RAM case */
1913 ptr = qemu_get_ram_ptr(addr1);
1914 memcpy(ptr, buf, l);
1915 invalidate_and_set_dirty(addr1, l);
1916 qemu_put_ram_ptr(ptr);
1917 }
1918 } else {
1919 if (!(memory_region_is_ram(section->mr) ||
1920 memory_region_is_romd(section->mr))) {
1921 hwaddr addr1;
1922 /* I/O case */
1923 addr1 = memory_region_section_addr(section, addr);
1924 if (l >= 4 && ((addr1 & 3) == 0)) {
1925 /* 32 bit read access */
1926 val = io_mem_read(section->mr, addr1, 4);
1927 stl_p(buf, val);
1928 l = 4;
1929 } else if (l >= 2 && ((addr1 & 1) == 0)) {
1930 /* 16 bit read access */
1931 val = io_mem_read(section->mr, addr1, 2);
1932 stw_p(buf, val);
1933 l = 2;
1934 } else {
1935 /* 8 bit read access */
1936 val = io_mem_read(section->mr, addr1, 1);
1937 stb_p(buf, val);
1938 l = 1;
1939 }
1940 } else {
1941 /* RAM case */
1942 ptr = qemu_get_ram_ptr(section->mr->ram_addr
1943 + memory_region_section_addr(section,
1944 addr));
1945 memcpy(buf, ptr, l);
1946 qemu_put_ram_ptr(ptr);
1947 }
1948 }
1949 len -= l;
1950 buf += l;
1951 addr += l;
1952 }
1953 }
1954
1955 void address_space_write(AddressSpace *as, hwaddr addr,
1956 const uint8_t *buf, int len)
1957 {
1958 address_space_rw(as, addr, (uint8_t *)buf, len, true);
1959 }
1960
1961 /**
1962 * address_space_read: read from an address space.
1963 *
1964 * @as: #AddressSpace to be accessed
1965 * @addr: address within that address space
1966 * @buf: buffer with the data transferred
1967 */
1968 void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
1969 {
1970 address_space_rw(as, addr, buf, len, false);
1971 }
1972
1973
1974 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
1975 int len, int is_write)
1976 {
1977 return address_space_rw(&address_space_memory, addr, buf, len, is_write);
1978 }
1979
1980 /* used for ROM loading : can write in RAM and ROM */
1981 void cpu_physical_memory_write_rom(hwaddr addr,
1982 const uint8_t *buf, int len)
1983 {
1984 AddressSpaceDispatch *d = address_space_memory.dispatch;
1985 int l;
1986 uint8_t *ptr;
1987 hwaddr page;
1988 MemoryRegionSection *section;
1989
1990 while (len > 0) {
1991 page = addr & TARGET_PAGE_MASK;
1992 l = (page + TARGET_PAGE_SIZE) - addr;
1993 if (l > len)
1994 l = len;
1995 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
1996
1997 if (!(memory_region_is_ram(section->mr) ||
1998 memory_region_is_romd(section->mr))) {
1999 /* do nothing */
2000 } else {
2001 unsigned long addr1;
2002 addr1 = memory_region_get_ram_addr(section->mr)
2003 + memory_region_section_addr(section, addr);
2004 /* ROM/RAM case */
2005 ptr = qemu_get_ram_ptr(addr1);
2006 memcpy(ptr, buf, l);
2007 invalidate_and_set_dirty(addr1, l);
2008 qemu_put_ram_ptr(ptr);
2009 }
2010 len -= l;
2011 buf += l;
2012 addr += l;
2013 }
2014 }
2015
2016 typedef struct {
2017 void *buffer;
2018 hwaddr addr;
2019 hwaddr len;
2020 } BounceBuffer;
2021
2022 static BounceBuffer bounce;
2023
2024 typedef struct MapClient {
2025 void *opaque;
2026 void (*callback)(void *opaque);
2027 QLIST_ENTRY(MapClient) link;
2028 } MapClient;
2029
2030 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2031 = QLIST_HEAD_INITIALIZER(map_client_list);
2032
2033 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2034 {
2035 MapClient *client = g_malloc(sizeof(*client));
2036
2037 client->opaque = opaque;
2038 client->callback = callback;
2039 QLIST_INSERT_HEAD(&map_client_list, client, link);
2040 return client;
2041 }
2042
2043 static void cpu_unregister_map_client(void *_client)
2044 {
2045 MapClient *client = (MapClient *)_client;
2046
2047 QLIST_REMOVE(client, link);
2048 g_free(client);
2049 }
2050
2051 static void cpu_notify_map_clients(void)
2052 {
2053 MapClient *client;
2054
2055 while (!QLIST_EMPTY(&map_client_list)) {
2056 client = QLIST_FIRST(&map_client_list);
2057 client->callback(client->opaque);
2058 cpu_unregister_map_client(client);
2059 }
2060 }
2061
2062 /* Map a physical memory region into a host virtual address.
2063 * May map a subset of the requested range, given by and returned in *plen.
2064 * May return NULL if resources needed to perform the mapping are exhausted.
2065 * Use only for reads OR writes - not for read-modify-write operations.
2066 * Use cpu_register_map_client() to know when retrying the map operation is
2067 * likely to succeed.
2068 */
2069 void *address_space_map(AddressSpace *as,
2070 hwaddr addr,
2071 hwaddr *plen,
2072 bool is_write)
2073 {
2074 AddressSpaceDispatch *d = as->dispatch;
2075 hwaddr len = *plen;
2076 hwaddr todo = 0;
2077 int l;
2078 hwaddr page;
2079 MemoryRegionSection *section;
2080 ram_addr_t raddr = RAM_ADDR_MAX;
2081 ram_addr_t rlen;
2082 void *ret;
2083
2084 while (len > 0) {
2085 page = addr & TARGET_PAGE_MASK;
2086 l = (page + TARGET_PAGE_SIZE) - addr;
2087 if (l > len)
2088 l = len;
2089 section = phys_page_find(d, page >> TARGET_PAGE_BITS);
2090
2091 if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
2092 if (todo || bounce.buffer) {
2093 break;
2094 }
2095 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
2096 bounce.addr = addr;
2097 bounce.len = l;
2098 if (!is_write) {
2099 address_space_read(as, addr, bounce.buffer, l);
2100 }
2101
2102 *plen = l;
2103 return bounce.buffer;
2104 }
2105 if (!todo) {
2106 raddr = memory_region_get_ram_addr(section->mr)
2107 + memory_region_section_addr(section, addr);
2108 }
2109
2110 len -= l;
2111 addr += l;
2112 todo += l;
2113 }
2114 rlen = todo;
2115 ret = qemu_ram_ptr_length(raddr, &rlen);
2116 *plen = rlen;
2117 return ret;
2118 }
2119
2120 /* Unmaps a memory region previously mapped by address_space_map().
2121 * Will also mark the memory as dirty if is_write == 1. access_len gives
2122 * the amount of memory that was actually read or written by the caller.
2123 */
2124 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2125 int is_write, hwaddr access_len)
2126 {
2127 if (buffer != bounce.buffer) {
2128 if (is_write) {
2129 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
2130 while (access_len) {
2131 unsigned l;
2132 l = TARGET_PAGE_SIZE;
2133 if (l > access_len)
2134 l = access_len;
2135 invalidate_and_set_dirty(addr1, l);
2136 addr1 += l;
2137 access_len -= l;
2138 }
2139 }
2140 if (xen_enabled()) {
2141 xen_invalidate_map_cache_entry(buffer);
2142 }
2143 return;
2144 }
2145 if (is_write) {
2146 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2147 }
2148 qemu_vfree(bounce.buffer);
2149 bounce.buffer = NULL;
2150 cpu_notify_map_clients();
2151 }
2152
2153 void *cpu_physical_memory_map(hwaddr addr,
2154 hwaddr *plen,
2155 int is_write)
2156 {
2157 return address_space_map(&address_space_memory, addr, plen, is_write);
2158 }
2159
2160 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2161 int is_write, hwaddr access_len)
2162 {
2163 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2164 }
2165
2166 /* warning: addr must be aligned */
2167 static inline uint32_t ldl_phys_internal(hwaddr addr,
2168 enum device_endian endian)
2169 {
2170 uint8_t *ptr;
2171 uint32_t val;
2172 MemoryRegionSection *section;
2173
2174 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2175
2176 if (!(memory_region_is_ram(section->mr) ||
2177 memory_region_is_romd(section->mr))) {
2178 /* I/O case */
2179 addr = memory_region_section_addr(section, addr);
2180 val = io_mem_read(section->mr, addr, 4);
2181 #if defined(TARGET_WORDS_BIGENDIAN)
2182 if (endian == DEVICE_LITTLE_ENDIAN) {
2183 val = bswap32(val);
2184 }
2185 #else
2186 if (endian == DEVICE_BIG_ENDIAN) {
2187 val = bswap32(val);
2188 }
2189 #endif
2190 } else {
2191 /* RAM case */
2192 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2193 & TARGET_PAGE_MASK)
2194 + memory_region_section_addr(section, addr));
2195 switch (endian) {
2196 case DEVICE_LITTLE_ENDIAN:
2197 val = ldl_le_p(ptr);
2198 break;
2199 case DEVICE_BIG_ENDIAN:
2200 val = ldl_be_p(ptr);
2201 break;
2202 default:
2203 val = ldl_p(ptr);
2204 break;
2205 }
2206 }
2207 return val;
2208 }
2209
2210 uint32_t ldl_phys(hwaddr addr)
2211 {
2212 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2213 }
2214
2215 uint32_t ldl_le_phys(hwaddr addr)
2216 {
2217 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2218 }
2219
2220 uint32_t ldl_be_phys(hwaddr addr)
2221 {
2222 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2223 }
2224
2225 /* warning: addr must be aligned */
2226 static inline uint64_t ldq_phys_internal(hwaddr addr,
2227 enum device_endian endian)
2228 {
2229 uint8_t *ptr;
2230 uint64_t val;
2231 MemoryRegionSection *section;
2232
2233 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2234
2235 if (!(memory_region_is_ram(section->mr) ||
2236 memory_region_is_romd(section->mr))) {
2237 /* I/O case */
2238 addr = memory_region_section_addr(section, addr);
2239
2240 /* XXX This is broken when device endian != cpu endian.
2241 Fix and add "endian" variable check */
2242 #ifdef TARGET_WORDS_BIGENDIAN
2243 val = io_mem_read(section->mr, addr, 4) << 32;
2244 val |= io_mem_read(section->mr, addr + 4, 4);
2245 #else
2246 val = io_mem_read(section->mr, addr, 4);
2247 val |= io_mem_read(section->mr, addr + 4, 4) << 32;
2248 #endif
2249 } else {
2250 /* RAM case */
2251 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2252 & TARGET_PAGE_MASK)
2253 + memory_region_section_addr(section, addr));
2254 switch (endian) {
2255 case DEVICE_LITTLE_ENDIAN:
2256 val = ldq_le_p(ptr);
2257 break;
2258 case DEVICE_BIG_ENDIAN:
2259 val = ldq_be_p(ptr);
2260 break;
2261 default:
2262 val = ldq_p(ptr);
2263 break;
2264 }
2265 }
2266 return val;
2267 }
2268
2269 uint64_t ldq_phys(hwaddr addr)
2270 {
2271 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2272 }
2273
2274 uint64_t ldq_le_phys(hwaddr addr)
2275 {
2276 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2277 }
2278
2279 uint64_t ldq_be_phys(hwaddr addr)
2280 {
2281 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2282 }
2283
2284 /* XXX: optimize */
2285 uint32_t ldub_phys(hwaddr addr)
2286 {
2287 uint8_t val;
2288 cpu_physical_memory_read(addr, &val, 1);
2289 return val;
2290 }
2291
2292 /* warning: addr must be aligned */
2293 static inline uint32_t lduw_phys_internal(hwaddr addr,
2294 enum device_endian endian)
2295 {
2296 uint8_t *ptr;
2297 uint64_t val;
2298 MemoryRegionSection *section;
2299
2300 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2301
2302 if (!(memory_region_is_ram(section->mr) ||
2303 memory_region_is_romd(section->mr))) {
2304 /* I/O case */
2305 addr = memory_region_section_addr(section, addr);
2306 val = io_mem_read(section->mr, addr, 2);
2307 #if defined(TARGET_WORDS_BIGENDIAN)
2308 if (endian == DEVICE_LITTLE_ENDIAN) {
2309 val = bswap16(val);
2310 }
2311 #else
2312 if (endian == DEVICE_BIG_ENDIAN) {
2313 val = bswap16(val);
2314 }
2315 #endif
2316 } else {
2317 /* RAM case */
2318 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2319 & TARGET_PAGE_MASK)
2320 + memory_region_section_addr(section, addr));
2321 switch (endian) {
2322 case DEVICE_LITTLE_ENDIAN:
2323 val = lduw_le_p(ptr);
2324 break;
2325 case DEVICE_BIG_ENDIAN:
2326 val = lduw_be_p(ptr);
2327 break;
2328 default:
2329 val = lduw_p(ptr);
2330 break;
2331 }
2332 }
2333 return val;
2334 }
2335
2336 uint32_t lduw_phys(hwaddr addr)
2337 {
2338 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2339 }
2340
2341 uint32_t lduw_le_phys(hwaddr addr)
2342 {
2343 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2344 }
2345
2346 uint32_t lduw_be_phys(hwaddr addr)
2347 {
2348 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2349 }
2350
2351 /* warning: addr must be aligned. The ram page is not masked as dirty
2352 and the code inside is not invalidated. It is useful if the dirty
2353 bits are used to track modified PTEs */
2354 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2355 {
2356 uint8_t *ptr;
2357 MemoryRegionSection *section;
2358
2359 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2360
2361 if (!memory_region_is_ram(section->mr) || section->readonly) {
2362 addr = memory_region_section_addr(section, addr);
2363 if (memory_region_is_ram(section->mr)) {
2364 section = &phys_sections[phys_section_rom];
2365 }
2366 io_mem_write(section->mr, addr, val, 4);
2367 } else {
2368 unsigned long addr1 = (memory_region_get_ram_addr(section->mr)
2369 & TARGET_PAGE_MASK)
2370 + memory_region_section_addr(section, addr);
2371 ptr = qemu_get_ram_ptr(addr1);
2372 stl_p(ptr, val);
2373
2374 if (unlikely(in_migration)) {
2375 if (!cpu_physical_memory_is_dirty(addr1)) {
2376 /* invalidate code */
2377 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2378 /* set dirty bit */
2379 cpu_physical_memory_set_dirty_flags(
2380 addr1, (0xff & ~CODE_DIRTY_FLAG));
2381 }
2382 }
2383 }
2384 }
2385
2386 void stq_phys_notdirty(hwaddr addr, uint64_t val)
2387 {
2388 uint8_t *ptr;
2389 MemoryRegionSection *section;
2390
2391 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2392
2393 if (!memory_region_is_ram(section->mr) || section->readonly) {
2394 addr = memory_region_section_addr(section, addr);
2395 if (memory_region_is_ram(section->mr)) {
2396 section = &phys_sections[phys_section_rom];
2397 }
2398 #ifdef TARGET_WORDS_BIGENDIAN
2399 io_mem_write(section->mr, addr, val >> 32, 4);
2400 io_mem_write(section->mr, addr + 4, (uint32_t)val, 4);
2401 #else
2402 io_mem_write(section->mr, addr, (uint32_t)val, 4);
2403 io_mem_write(section->mr, addr + 4, val >> 32, 4);
2404 #endif
2405 } else {
2406 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section->mr)
2407 & TARGET_PAGE_MASK)
2408 + memory_region_section_addr(section, addr));
2409 stq_p(ptr, val);
2410 }
2411 }
2412
2413 /* warning: addr must be aligned */
2414 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2415 enum device_endian endian)
2416 {
2417 uint8_t *ptr;
2418 MemoryRegionSection *section;
2419
2420 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2421
2422 if (!memory_region_is_ram(section->mr) || section->readonly) {
2423 addr = memory_region_section_addr(section, addr);
2424 if (memory_region_is_ram(section->mr)) {
2425 section = &phys_sections[phys_section_rom];
2426 }
2427 #if defined(TARGET_WORDS_BIGENDIAN)
2428 if (endian == DEVICE_LITTLE_ENDIAN) {
2429 val = bswap32(val);
2430 }
2431 #else
2432 if (endian == DEVICE_BIG_ENDIAN) {
2433 val = bswap32(val);
2434 }
2435 #endif
2436 io_mem_write(section->mr, addr, val, 4);
2437 } else {
2438 unsigned long addr1;
2439 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2440 + memory_region_section_addr(section, addr);
2441 /* RAM case */
2442 ptr = qemu_get_ram_ptr(addr1);
2443 switch (endian) {
2444 case DEVICE_LITTLE_ENDIAN:
2445 stl_le_p(ptr, val);
2446 break;
2447 case DEVICE_BIG_ENDIAN:
2448 stl_be_p(ptr, val);
2449 break;
2450 default:
2451 stl_p(ptr, val);
2452 break;
2453 }
2454 invalidate_and_set_dirty(addr1, 4);
2455 }
2456 }
2457
2458 void stl_phys(hwaddr addr, uint32_t val)
2459 {
2460 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2461 }
2462
2463 void stl_le_phys(hwaddr addr, uint32_t val)
2464 {
2465 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2466 }
2467
2468 void stl_be_phys(hwaddr addr, uint32_t val)
2469 {
2470 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2471 }
2472
2473 /* XXX: optimize */
2474 void stb_phys(hwaddr addr, uint32_t val)
2475 {
2476 uint8_t v = val;
2477 cpu_physical_memory_write(addr, &v, 1);
2478 }
2479
2480 /* warning: addr must be aligned */
2481 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2482 enum device_endian endian)
2483 {
2484 uint8_t *ptr;
2485 MemoryRegionSection *section;
2486
2487 section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
2488
2489 if (!memory_region_is_ram(section->mr) || section->readonly) {
2490 addr = memory_region_section_addr(section, addr);
2491 if (memory_region_is_ram(section->mr)) {
2492 section = &phys_sections[phys_section_rom];
2493 }
2494 #if defined(TARGET_WORDS_BIGENDIAN)
2495 if (endian == DEVICE_LITTLE_ENDIAN) {
2496 val = bswap16(val);
2497 }
2498 #else
2499 if (endian == DEVICE_BIG_ENDIAN) {
2500 val = bswap16(val);
2501 }
2502 #endif
2503 io_mem_write(section->mr, addr, val, 2);
2504 } else {
2505 unsigned long addr1;
2506 addr1 = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
2507 + memory_region_section_addr(section, addr);
2508 /* RAM case */
2509 ptr = qemu_get_ram_ptr(addr1);
2510 switch (endian) {
2511 case DEVICE_LITTLE_ENDIAN:
2512 stw_le_p(ptr, val);
2513 break;
2514 case DEVICE_BIG_ENDIAN:
2515 stw_be_p(ptr, val);
2516 break;
2517 default:
2518 stw_p(ptr, val);
2519 break;
2520 }
2521 invalidate_and_set_dirty(addr1, 2);
2522 }
2523 }
2524
2525 void stw_phys(hwaddr addr, uint32_t val)
2526 {
2527 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2528 }
2529
2530 void stw_le_phys(hwaddr addr, uint32_t val)
2531 {
2532 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2533 }
2534
2535 void stw_be_phys(hwaddr addr, uint32_t val)
2536 {
2537 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2538 }
2539
2540 /* XXX: optimize */
2541 void stq_phys(hwaddr addr, uint64_t val)
2542 {
2543 val = tswap64(val);
2544 cpu_physical_memory_write(addr, &val, 8);
2545 }
2546
2547 void stq_le_phys(hwaddr addr, uint64_t val)
2548 {
2549 val = cpu_to_le64(val);
2550 cpu_physical_memory_write(addr, &val, 8);
2551 }
2552
2553 void stq_be_phys(hwaddr addr, uint64_t val)
2554 {
2555 val = cpu_to_be64(val);
2556 cpu_physical_memory_write(addr, &val, 8);
2557 }
2558
2559 /* virtual memory access for debug (includes writing to ROM) */
2560 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
2561 uint8_t *buf, int len, int is_write)
2562 {
2563 int l;
2564 hwaddr phys_addr;
2565 target_ulong page;
2566
2567 while (len > 0) {
2568 page = addr & TARGET_PAGE_MASK;
2569 phys_addr = cpu_get_phys_page_debug(env, page);
2570 /* if no physical page mapped, return an error */
2571 if (phys_addr == -1)
2572 return -1;
2573 l = (page + TARGET_PAGE_SIZE) - addr;
2574 if (l > len)
2575 l = len;
2576 phys_addr += (addr & ~TARGET_PAGE_MASK);
2577 if (is_write)
2578 cpu_physical_memory_write_rom(phys_addr, buf, l);
2579 else
2580 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2581 len -= l;
2582 buf += l;
2583 addr += l;
2584 }
2585 return 0;
2586 }
2587 #endif
2588
2589 #if !defined(CONFIG_USER_ONLY)
2590
2591 /*
2592 * A helper function for the _utterly broken_ virtio device model to find out if
2593 * it's running on a big endian machine. Don't do this at home kids!
2594 */
2595 bool virtio_is_big_endian(void);
2596 bool virtio_is_big_endian(void)
2597 {
2598 #if defined(TARGET_WORDS_BIGENDIAN)
2599 return true;
2600 #else
2601 return false;
2602 #endif
2603 }
2604
2605 #endif
2606
2607 #ifndef CONFIG_USER_ONLY
2608 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2609 {
2610 MemoryRegionSection *section;
2611
2612 section = phys_page_find(address_space_memory.dispatch,
2613 phys_addr >> TARGET_PAGE_BITS);
2614
2615 return !(memory_region_is_ram(section->mr) ||
2616 memory_region_is_romd(section->mr));
2617 }
2618 #endif
Ray Wang's QEMU Repository