?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
cpu: Move mem_io_{pc,vaddr} fields from CPU_COMMON to CPUState
[qemu/cris-port.git] / exec.c
blob6666f6d396adbfc69ee9c41e5a00a145bb87b56a
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifndef _WIN32
21 #include <sys/types.h>
22 #include <sys/mman.h>
23 #endif
24
25 #include "qemu-common.h"
26 #include "cpu.h"
27 #include "tcg.h"
28 #include "hw/hw.h"
29 #include "hw/qdev.h"
30 #include "qemu/osdep.h"
31 #include "sysemu/kvm.h"
32 #include "sysemu/sysemu.h"
33 #include "hw/xen/xen.h"
34 #include "qemu/timer.h"
35 #include "qemu/config-file.h"
36 #include "exec/memory.h"
37 #include "sysemu/dma.h"
38 #include "exec/address-spaces.h"
39 #if defined(CONFIG_USER_ONLY)
40 #include <qemu.h>
41 #else /* !CONFIG_USER_ONLY */
42 #include "sysemu/xen-mapcache.h"
43 #include "trace.h"
44 #endif
45 #include "exec/cpu-all.h"
46
47 #include "exec/cputlb.h"
48 #include "translate-all.h"
49
50 #include "exec/memory-internal.h"
51 #include "exec/ram_addr.h"
52 #include "qemu/cache-utils.h"
53
54 #include "qemu/range.h"
55
56 //#define DEBUG_SUBPAGE
57
58 #if !defined(CONFIG_USER_ONLY)
59 static bool in_migration;
60
61 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
62
63 static MemoryRegion *system_memory;
64 static MemoryRegion *system_io;
65
66 AddressSpace address_space_io;
67 AddressSpace address_space_memory;
68
69 MemoryRegion io_mem_rom, io_mem_notdirty;
70 static MemoryRegion io_mem_unassigned;
71
72 #endif
73
74 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
75 /* current CPU in the current thread. It is only valid inside
76 cpu_exec() */
77 DEFINE_TLS(CPUState *, current_cpu);
78 /* 0 = Do not count executed instructions.
79 1 = Precise instruction counting.
80 2 = Adaptive rate instruction counting. */
81 int use_icount;
82
83 #if !defined(CONFIG_USER_ONLY)
84
85 typedef struct PhysPageEntry PhysPageEntry;
86
87 struct PhysPageEntry {
88 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
89 uint32_t skip : 6;
90 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
91 uint32_t ptr : 26;
92 };
93
94 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
95
96 /* Size of the L2 (and L3, etc) page tables. */
97 #define ADDR_SPACE_BITS 64
98
99 #define P_L2_BITS 9
100 #define P_L2_SIZE (1 << P_L2_BITS)
101
102 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
103
104 typedef PhysPageEntry Node[P_L2_SIZE];
105
106 typedef struct PhysPageMap {
107 unsigned sections_nb;
108 unsigned sections_nb_alloc;
109 unsigned nodes_nb;
110 unsigned nodes_nb_alloc;
111 Node *nodes;
112 MemoryRegionSection *sections;
113 } PhysPageMap;
114
115 struct AddressSpaceDispatch {
116 /* This is a multi-level map on the physical address space.
117 * The bottom level has pointers to MemoryRegionSections.
118 */
119 PhysPageEntry phys_map;
120 PhysPageMap map;
121 AddressSpace *as;
122 };
123
124 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
125 typedef struct subpage_t {
126 MemoryRegion iomem;
127 AddressSpace *as;
128 hwaddr base;
129 uint16_t sub_section[TARGET_PAGE_SIZE];
130 } subpage_t;
131
132 #define PHYS_SECTION_UNASSIGNED 0
133 #define PHYS_SECTION_NOTDIRTY 1
134 #define PHYS_SECTION_ROM 2
135 #define PHYS_SECTION_WATCH 3
136
137 static void io_mem_init(void);
138 static void memory_map_init(void);
139 static void tcg_commit(MemoryListener *listener);
140
141 static MemoryRegion io_mem_watch;
142 #endif
143
144 #if !defined(CONFIG_USER_ONLY)
145
146 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
147 {
148 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
149 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
150 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
151 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
152 }
153 }
154
155 static uint32_t phys_map_node_alloc(PhysPageMap *map)
156 {
157 unsigned i;
158 uint32_t ret;
159
160 ret = map->nodes_nb++;
161 assert(ret != PHYS_MAP_NODE_NIL);
162 assert(ret != map->nodes_nb_alloc);
163 for (i = 0; i < P_L2_SIZE; ++i) {
164 map->nodes[ret][i].skip = 1;
165 map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
166 }
167 return ret;
168 }
169
170 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
171 hwaddr *index, hwaddr *nb, uint16_t leaf,
172 int level)
173 {
174 PhysPageEntry *p;
175 int i;
176 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
177
178 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
179 lp->ptr = phys_map_node_alloc(map);
180 p = map->nodes[lp->ptr];
181 if (level == 0) {
182 for (i = 0; i < P_L2_SIZE; i++) {
183 p[i].skip = 0;
184 p[i].ptr = PHYS_SECTION_UNASSIGNED;
185 }
186 }
187 } else {
188 p = map->nodes[lp->ptr];
189 }
190 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
191
192 while (*nb && lp < &p[P_L2_SIZE]) {
193 if ((*index & (step - 1)) == 0 && *nb >= step) {
194 lp->skip = 0;
195 lp->ptr = leaf;
196 *index += step;
197 *nb -= step;
198 } else {
199 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
200 }
201 ++lp;
202 }
203 }
204
205 static void phys_page_set(AddressSpaceDispatch *d,
206 hwaddr index, hwaddr nb,
207 uint16_t leaf)
208 {
209 /* Wildly overreserve - it doesn't matter much. */
210 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
211
212 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
213 }
214
215 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
216 * and update our entry so we can skip it and go directly to the destination.
217 */
218 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
219 {
220 unsigned valid_ptr = P_L2_SIZE;
221 int valid = 0;
222 PhysPageEntry *p;
223 int i;
224
225 if (lp->ptr == PHYS_MAP_NODE_NIL) {
226 return;
227 }
228
229 p = nodes[lp->ptr];
230 for (i = 0; i < P_L2_SIZE; i++) {
231 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
232 continue;
233 }
234
235 valid_ptr = i;
236 valid++;
237 if (p[i].skip) {
238 phys_page_compact(&p[i], nodes, compacted);
239 }
240 }
241
242 /* We can only compress if there's only one child. */
243 if (valid != 1) {
244 return;
245 }
246
247 assert(valid_ptr < P_L2_SIZE);
248
249 /* Don't compress if it won't fit in the # of bits we have. */
250 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
251 return;
252 }
253
254 lp->ptr = p[valid_ptr].ptr;
255 if (!p[valid_ptr].skip) {
256 /* If our only child is a leaf, make this a leaf. */
257 /* By design, we should have made this node a leaf to begin with so we
258 * should never reach here.
259 * But since it's so simple to handle this, let's do it just in case we
260 * change this rule.
261 */
262 lp->skip = 0;
263 } else {
264 lp->skip += p[valid_ptr].skip;
265 }
266 }
267
268 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
269 {
270 DECLARE_BITMAP(compacted, nodes_nb);
271
272 if (d->phys_map.skip) {
273 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
274 }
275 }
276
277 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
278 Node *nodes, MemoryRegionSection *sections)
279 {
280 PhysPageEntry *p;
281 hwaddr index = addr >> TARGET_PAGE_BITS;
282 int i;
283
284 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
285 if (lp.ptr == PHYS_MAP_NODE_NIL) {
286 return &sections[PHYS_SECTION_UNASSIGNED];
287 }
288 p = nodes[lp.ptr];
289 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
290 }
291
292 if (sections[lp.ptr].size.hi ||
293 range_covers_byte(sections[lp.ptr].offset_within_address_space,
294 sections[lp.ptr].size.lo, addr)) {
295 return &sections[lp.ptr];
296 } else {
297 return &sections[PHYS_SECTION_UNASSIGNED];
298 }
299 }
300
301 bool memory_region_is_unassigned(MemoryRegion *mr)
302 {
303 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
304 && mr != &io_mem_watch;
305 }
306
307 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
308 hwaddr addr,
309 bool resolve_subpage)
310 {
311 MemoryRegionSection *section;
312 subpage_t *subpage;
313
314 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
315 if (resolve_subpage && section->mr->subpage) {
316 subpage = container_of(section->mr, subpage_t, iomem);
317 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
318 }
319 return section;
320 }
321
322 static MemoryRegionSection *
323 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
324 hwaddr *plen, bool resolve_subpage)
325 {
326 MemoryRegionSection *section;
327 Int128 diff;
328
329 section = address_space_lookup_region(d, addr, resolve_subpage);
330 /* Compute offset within MemoryRegionSection */
331 addr -= section->offset_within_address_space;
332
333 /* Compute offset within MemoryRegion */
334 *xlat = addr + section->offset_within_region;
335
336 diff = int128_sub(section->mr->size, int128_make64(addr));
337 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
338 return section;
339 }
340
341 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
342 {
343 if (memory_region_is_ram(mr)) {
344 return !(is_write && mr->readonly);
345 }
346 if (memory_region_is_romd(mr)) {
347 return !is_write;
348 }
349
350 return false;
351 }
352
353 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
354 hwaddr *xlat, hwaddr *plen,
355 bool is_write)
356 {
357 IOMMUTLBEntry iotlb;
358 MemoryRegionSection *section;
359 MemoryRegion *mr;
360 hwaddr len = *plen;
361
362 for (;;) {
363 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
364 mr = section->mr;
365
366 if (!mr->iommu_ops) {
367 break;
368 }
369
370 iotlb = mr->iommu_ops->translate(mr, addr);
371 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
372 | (addr & iotlb.addr_mask));
373 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
374 if (!(iotlb.perm & (1 << is_write))) {
375 mr = &io_mem_unassigned;
376 break;
377 }
378
379 as = iotlb.target_as;
380 }
381
382 if (memory_access_is_direct(mr, is_write)) {
383 hwaddr page = ((addr & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE) - addr;
384 len = MIN(page, len);
385 }
386
387 *plen = len;
388 *xlat = addr;
389 return mr;
390 }
391
392 MemoryRegionSection *
393 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
394 hwaddr *plen)
395 {
396 MemoryRegionSection *section;
397 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
398
399 assert(!section->mr->iommu_ops);
400 return section;
401 }
402 #endif
403
404 void cpu_exec_init_all(void)
405 {
406 #if !defined(CONFIG_USER_ONLY)
407 qemu_mutex_init(&ram_list.mutex);
408 memory_map_init();
409 io_mem_init();
410 #endif
411 }
412
413 #if !defined(CONFIG_USER_ONLY)
414
415 static int cpu_common_post_load(void *opaque, int version_id)
416 {
417 CPUState *cpu = opaque;
418
419 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
420 version_id is increased. */
421 cpu->interrupt_request &= ~0x01;
422 tlb_flush(cpu->env_ptr, 1);
423
424 return 0;
425 }
426
427 const VMStateDescription vmstate_cpu_common = {
428 .name = "cpu_common",
429 .version_id = 1,
430 .minimum_version_id = 1,
431 .minimum_version_id_old = 1,
432 .post_load = cpu_common_post_load,
433 .fields = (VMStateField []) {
434 VMSTATE_UINT32(halted, CPUState),
435 VMSTATE_UINT32(interrupt_request, CPUState),
436 VMSTATE_END_OF_LIST()
437 }
438 };
439
440 #endif
441
442 CPUState *qemu_get_cpu(int index)
443 {
444 CPUState *cpu;
445
446 CPU_FOREACH(cpu) {
447 if (cpu->cpu_index == index) {
448 return cpu;
449 }
450 }
451
452 return NULL;
453 }
454
455 #if !defined(CONFIG_USER_ONLY)
456 void tcg_cpu_address_space_init(CPUState *cpu, AddressSpace *as)
457 {
458 /* We only support one address space per cpu at the moment. */
459 assert(cpu->as == as);
460
461 if (cpu->tcg_as_listener) {
462 memory_listener_unregister(cpu->tcg_as_listener);
463 } else {
464 cpu->tcg_as_listener = g_new0(MemoryListener, 1);
465 }
466 cpu->tcg_as_listener->commit = tcg_commit;
467 memory_listener_register(cpu->tcg_as_listener, as);
468 }
469 #endif
470
471 void cpu_exec_init(CPUArchState *env)
472 {
473 CPUState *cpu = ENV_GET_CPU(env);
474 CPUClass *cc = CPU_GET_CLASS(cpu);
475 CPUState *some_cpu;
476 int cpu_index;
477
478 #if defined(CONFIG_USER_ONLY)
479 cpu_list_lock();
480 #endif
481 cpu_index = 0;
482 CPU_FOREACH(some_cpu) {
483 cpu_index++;
484 }
485 cpu->cpu_index = cpu_index;
486 cpu->numa_node = 0;
487 QTAILQ_INIT(&env->breakpoints);
488 QTAILQ_INIT(&env->watchpoints);
489 #ifndef CONFIG_USER_ONLY
490 cpu->as = &address_space_memory;
491 cpu->thread_id = qemu_get_thread_id();
492 #endif
493 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
494 #if defined(CONFIG_USER_ONLY)
495 cpu_list_unlock();
496 #endif
497 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
498 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
499 }
500 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
501 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
502 cpu_save, cpu_load, env);
503 assert(cc->vmsd == NULL);
504 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
505 #endif
506 if (cc->vmsd != NULL) {
507 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
508 }
509 }
510
511 #if defined(TARGET_HAS_ICE)
512 #if defined(CONFIG_USER_ONLY)
513 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
514 {
515 tb_invalidate_phys_page_range(pc, pc + 1, 0);
516 }
517 #else
518 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
519 {
520 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
521 if (phys != -1) {
522 tb_invalidate_phys_addr(cpu->as,
523 phys | (pc & ~TARGET_PAGE_MASK));
524 }
525 }
526 #endif
527 #endif /* TARGET_HAS_ICE */
528
529 #if defined(CONFIG_USER_ONLY)
530 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
531
532 {
533 }
534
535 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
536 int flags, CPUWatchpoint **watchpoint)
537 {
538 return -ENOSYS;
539 }
540 #else
541 /* Add a watchpoint. */
542 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
543 int flags, CPUWatchpoint **watchpoint)
544 {
545 target_ulong len_mask = ~(len - 1);
546 CPUWatchpoint *wp;
547
548 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
549 if ((len & (len - 1)) || (addr & ~len_mask) ||
550 len == 0 || len > TARGET_PAGE_SIZE) {
551 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
552 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
553 return -EINVAL;
554 }
555 wp = g_malloc(sizeof(*wp));
556
557 wp->vaddr = addr;
558 wp->len_mask = len_mask;
559 wp->flags = flags;
560
561 /* keep all GDB-injected watchpoints in front */
562 if (flags & BP_GDB)
563 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
564 else
565 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
566
567 tlb_flush_page(env, addr);
568
569 if (watchpoint)
570 *watchpoint = wp;
571 return 0;
572 }
573
574 /* Remove a specific watchpoint. */
575 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
576 int flags)
577 {
578 target_ulong len_mask = ~(len - 1);
579 CPUWatchpoint *wp;
580
581 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
582 if (addr == wp->vaddr && len_mask == wp->len_mask
583 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
584 cpu_watchpoint_remove_by_ref(env, wp);
585 return 0;
586 }
587 }
588 return -ENOENT;
589 }
590
591 /* Remove a specific watchpoint by reference. */
592 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
593 {
594 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
595
596 tlb_flush_page(env, watchpoint->vaddr);
597
598 g_free(watchpoint);
599 }
600
601 /* Remove all matching watchpoints. */
602 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
603 {
604 CPUWatchpoint *wp, *next;
605
606 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
607 if (wp->flags & mask)
608 cpu_watchpoint_remove_by_ref(env, wp);
609 }
610 }
611 #endif
612
613 /* Add a breakpoint. */
614 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
615 CPUBreakpoint **breakpoint)
616 {
617 #if defined(TARGET_HAS_ICE)
618 CPUBreakpoint *bp;
619
620 bp = g_malloc(sizeof(*bp));
621
622 bp->pc = pc;
623 bp->flags = flags;
624
625 /* keep all GDB-injected breakpoints in front */
626 if (flags & BP_GDB) {
627 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
628 } else {
629 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
630 }
631
632 breakpoint_invalidate(ENV_GET_CPU(env), pc);
633
634 if (breakpoint) {
635 *breakpoint = bp;
636 }
637 return 0;
638 #else
639 return -ENOSYS;
640 #endif
641 }
642
643 /* Remove a specific breakpoint. */
644 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
645 {
646 #if defined(TARGET_HAS_ICE)
647 CPUBreakpoint *bp;
648
649 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
650 if (bp->pc == pc && bp->flags == flags) {
651 cpu_breakpoint_remove_by_ref(env, bp);
652 return 0;
653 }
654 }
655 return -ENOENT;
656 #else
657 return -ENOSYS;
658 #endif
659 }
660
661 /* Remove a specific breakpoint by reference. */
662 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
663 {
664 #if defined(TARGET_HAS_ICE)
665 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
666
667 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
668
669 g_free(breakpoint);
670 #endif
671 }
672
673 /* Remove all matching breakpoints. */
674 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
675 {
676 #if defined(TARGET_HAS_ICE)
677 CPUBreakpoint *bp, *next;
678
679 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
680 if (bp->flags & mask)
681 cpu_breakpoint_remove_by_ref(env, bp);
682 }
683 #endif
684 }
685
686 /* enable or disable single step mode. EXCP_DEBUG is returned by the
687 CPU loop after each instruction */
688 void cpu_single_step(CPUState *cpu, int enabled)
689 {
690 #if defined(TARGET_HAS_ICE)
691 if (cpu->singlestep_enabled != enabled) {
692 cpu->singlestep_enabled = enabled;
693 if (kvm_enabled()) {
694 kvm_update_guest_debug(cpu, 0);
695 } else {
696 /* must flush all the translated code to avoid inconsistencies */
697 /* XXX: only flush what is necessary */
698 CPUArchState *env = cpu->env_ptr;
699 tb_flush(env);
700 }
701 }
702 #endif
703 }
704
705 void cpu_abort(CPUArchState *env, const char *fmt, ...)
706 {
707 CPUState *cpu = ENV_GET_CPU(env);
708 va_list ap;
709 va_list ap2;
710
711 va_start(ap, fmt);
712 va_copy(ap2, ap);
713 fprintf(stderr, "qemu: fatal: ");
714 vfprintf(stderr, fmt, ap);
715 fprintf(stderr, "\n");
716 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
717 if (qemu_log_enabled()) {
718 qemu_log("qemu: fatal: ");
719 qemu_log_vprintf(fmt, ap2);
720 qemu_log("\n");
721 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
722 qemu_log_flush();
723 qemu_log_close();
724 }
725 va_end(ap2);
726 va_end(ap);
727 #if defined(CONFIG_USER_ONLY)
728 {
729 struct sigaction act;
730 sigfillset(&act.sa_mask);
731 act.sa_handler = SIG_DFL;
732 sigaction(SIGABRT, &act, NULL);
733 }
734 #endif
735 abort();
736 }
737
738 #if !defined(CONFIG_USER_ONLY)
739 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
740 {
741 RAMBlock *block;
742
743 /* The list is protected by the iothread lock here. */
744 block = ram_list.mru_block;
745 if (block && addr - block->offset < block->length) {
746 goto found;
747 }
748 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
749 if (addr - block->offset < block->length) {
750 goto found;
751 }
752 }
753
754 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
755 abort();
756
757 found:
758 ram_list.mru_block = block;
759 return block;
760 }
761
762 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t length)
763 {
764 ram_addr_t start1;
765 RAMBlock *block;
766 ram_addr_t end;
767
768 end = TARGET_PAGE_ALIGN(start + length);
769 start &= TARGET_PAGE_MASK;
770
771 block = qemu_get_ram_block(start);
772 assert(block == qemu_get_ram_block(end - 1));
773 start1 = (uintptr_t)block->host + (start - block->offset);
774 cpu_tlb_reset_dirty_all(start1, length);
775 }
776
777 /* Note: start and end must be within the same ram block. */
778 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t length,
779 unsigned client)
780 {
781 if (length == 0)
782 return;
783 cpu_physical_memory_clear_dirty_range(start, length, client);
784
785 if (tcg_enabled()) {
786 tlb_reset_dirty_range_all(start, length);
787 }
788 }
789
790 static void cpu_physical_memory_set_dirty_tracking(bool enable)
791 {
792 in_migration = enable;
793 }
794
795 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
796 MemoryRegionSection *section,
797 target_ulong vaddr,
798 hwaddr paddr, hwaddr xlat,
799 int prot,
800 target_ulong *address)
801 {
802 hwaddr iotlb;
803 CPUWatchpoint *wp;
804
805 if (memory_region_is_ram(section->mr)) {
806 /* Normal RAM. */
807 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
808 + xlat;
809 if (!section->readonly) {
810 iotlb |= PHYS_SECTION_NOTDIRTY;
811 } else {
812 iotlb |= PHYS_SECTION_ROM;
813 }
814 } else {
815 iotlb = section - section->address_space->dispatch->map.sections;
816 iotlb += xlat;
817 }
818
819 /* Make accesses to pages with watchpoints go via the
820 watchpoint trap routines. */
821 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
822 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
823 /* Avoid trapping reads of pages with a write breakpoint. */
824 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
825 iotlb = PHYS_SECTION_WATCH + paddr;
826 *address |= TLB_MMIO;
827 break;
828 }
829 }
830 }
831
832 return iotlb;
833 }
834 #endif /* defined(CONFIG_USER_ONLY) */
835
836 #if !defined(CONFIG_USER_ONLY)
837
838 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
839 uint16_t section);
840 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
841
842 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
843
844 /*
845 * Set a custom physical guest memory alloator.
846 * Accelerators with unusual needs may need this. Hopefully, we can
847 * get rid of it eventually.
848 */
849 void phys_mem_set_alloc(void *(*alloc)(size_t))
850 {
851 phys_mem_alloc = alloc;
852 }
853
854 static uint16_t phys_section_add(PhysPageMap *map,
855 MemoryRegionSection *section)
856 {
857 /* The physical section number is ORed with a page-aligned
858 * pointer to produce the iotlb entries. Thus it should
859 * never overflow into the page-aligned value.
860 */
861 assert(map->sections_nb < TARGET_PAGE_SIZE);
862
863 if (map->sections_nb == map->sections_nb_alloc) {
864 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
865 map->sections = g_renew(MemoryRegionSection, map->sections,
866 map->sections_nb_alloc);
867 }
868 map->sections[map->sections_nb] = *section;
869 memory_region_ref(section->mr);
870 return map->sections_nb++;
871 }
872
873 static void phys_section_destroy(MemoryRegion *mr)
874 {
875 memory_region_unref(mr);
876
877 if (mr->subpage) {
878 subpage_t *subpage = container_of(mr, subpage_t, iomem);
879 memory_region_destroy(&subpage->iomem);
880 g_free(subpage);
881 }
882 }
883
884 static void phys_sections_free(PhysPageMap *map)
885 {
886 while (map->sections_nb > 0) {
887 MemoryRegionSection *section = &map->sections[--map->sections_nb];
888 phys_section_destroy(section->mr);
889 }
890 g_free(map->sections);
891 g_free(map->nodes);
892 }
893
894 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
895 {
896 subpage_t *subpage;
897 hwaddr base = section->offset_within_address_space
898 & TARGET_PAGE_MASK;
899 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
900 d->map.nodes, d->map.sections);
901 MemoryRegionSection subsection = {
902 .offset_within_address_space = base,
903 .size = int128_make64(TARGET_PAGE_SIZE),
904 };
905 hwaddr start, end;
906
907 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
908
909 if (!(existing->mr->subpage)) {
910 subpage = subpage_init(d->as, base);
911 subsection.address_space = d->as;
912 subsection.mr = &subpage->iomem;
913 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
914 phys_section_add(&d->map, &subsection));
915 } else {
916 subpage = container_of(existing->mr, subpage_t, iomem);
917 }
918 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
919 end = start + int128_get64(section->size) - 1;
920 subpage_register(subpage, start, end,
921 phys_section_add(&d->map, section));
922 }
923
924
925 static void register_multipage(AddressSpaceDispatch *d,
926 MemoryRegionSection *section)
927 {
928 hwaddr start_addr = section->offset_within_address_space;
929 uint16_t section_index = phys_section_add(&d->map, section);
930 uint64_t num_pages = int128_get64(int128_rshift(section->size,
931 TARGET_PAGE_BITS));
932
933 assert(num_pages);
934 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
935 }
936
937 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
938 {
939 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
940 AddressSpaceDispatch *d = as->next_dispatch;
941 MemoryRegionSection now = *section, remain = *section;
942 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
943
944 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
945 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
946 - now.offset_within_address_space;
947
948 now.size = int128_min(int128_make64(left), now.size);
949 register_subpage(d, &now);
950 } else {
951 now.size = int128_zero();
952 }
953 while (int128_ne(remain.size, now.size)) {
954 remain.size = int128_sub(remain.size, now.size);
955 remain.offset_within_address_space += int128_get64(now.size);
956 remain.offset_within_region += int128_get64(now.size);
957 now = remain;
958 if (int128_lt(remain.size, page_size)) {
959 register_subpage(d, &now);
960 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
961 now.size = page_size;
962 register_subpage(d, &now);
963 } else {
964 now.size = int128_and(now.size, int128_neg(page_size));
965 register_multipage(d, &now);
966 }
967 }
968 }
969
970 void qemu_flush_coalesced_mmio_buffer(void)
971 {
972 if (kvm_enabled())
973 kvm_flush_coalesced_mmio_buffer();
974 }
975
976 void qemu_mutex_lock_ramlist(void)
977 {
978 qemu_mutex_lock(&ram_list.mutex);
979 }
980
981 void qemu_mutex_unlock_ramlist(void)
982 {
983 qemu_mutex_unlock(&ram_list.mutex);
984 }
985
986 #ifdef __linux__
987
988 #include <sys/vfs.h>
989
990 #define HUGETLBFS_MAGIC 0x958458f6
991
992 static long gethugepagesize(const char *path)
993 {
994 struct statfs fs;
995 int ret;
996
997 do {
998 ret = statfs(path, &fs);
999 } while (ret != 0 && errno == EINTR);
1000
1001 if (ret != 0) {
1002 perror(path);
1003 return 0;
1004 }
1005
1006 if (fs.f_type != HUGETLBFS_MAGIC)
1007 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
1008
1009 return fs.f_bsize;
1010 }
1011
1012 static sigjmp_buf sigjump;
1013
1014 static void sigbus_handler(int signal)
1015 {
1016 siglongjmp(sigjump, 1);
1017 }
1018
1019 static void *file_ram_alloc(RAMBlock *block,
1020 ram_addr_t memory,
1021 const char *path)
1022 {
1023 char *filename;
1024 char *sanitized_name;
1025 char *c;
1026 void *area;
1027 int fd;
1028 unsigned long hpagesize;
1029
1030 hpagesize = gethugepagesize(path);
1031 if (!hpagesize) {
1032 goto error;
1033 }
1034
1035 if (memory < hpagesize) {
1036 return NULL;
1037 }
1038
1039 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1040 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
1041 goto error;
1042 }
1043
1044 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1045 sanitized_name = g_strdup(block->mr->name);
1046 for (c = sanitized_name; *c != '\0'; c++) {
1047 if (*c == '/')
1048 *c = '_';
1049 }
1050
1051 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1052 sanitized_name);
1053 g_free(sanitized_name);
1054
1055 fd = mkstemp(filename);
1056 if (fd < 0) {
1057 perror("unable to create backing store for hugepages");
1058 g_free(filename);
1059 goto error;
1060 }
1061 unlink(filename);
1062 g_free(filename);
1063
1064 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1065
1066 /*
1067 * ftruncate is not supported by hugetlbfs in older
1068 * hosts, so don't bother bailing out on errors.
1069 * If anything goes wrong with it under other filesystems,
1070 * mmap will fail.
1071 */
1072 if (ftruncate(fd, memory))
1073 perror("ftruncate");
1074
1075 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1076 if (area == MAP_FAILED) {
1077 perror("file_ram_alloc: can't mmap RAM pages");
1078 close(fd);
1079 goto error;
1080 }
1081
1082 if (mem_prealloc) {
1083 int ret, i;
1084 struct sigaction act, oldact;
1085 sigset_t set, oldset;
1086
1087 memset(&act, 0, sizeof(act));
1088 act.sa_handler = &sigbus_handler;
1089 act.sa_flags = 0;
1090
1091 ret = sigaction(SIGBUS, &act, &oldact);
1092 if (ret) {
1093 perror("file_ram_alloc: failed to install signal handler");
1094 exit(1);
1095 }
1096
1097 /* unblock SIGBUS */
1098 sigemptyset(&set);
1099 sigaddset(&set, SIGBUS);
1100 pthread_sigmask(SIG_UNBLOCK, &set, &oldset);
1101
1102 if (sigsetjmp(sigjump, 1)) {
1103 fprintf(stderr, "file_ram_alloc: failed to preallocate pages\n");
1104 exit(1);
1105 }
1106
1107 /* MAP_POPULATE silently ignores failures */
1108 for (i = 0; i < (memory/hpagesize); i++) {
1109 memset(area + (hpagesize*i), 0, 1);
1110 }
1111
1112 ret = sigaction(SIGBUS, &oldact, NULL);
1113 if (ret) {
1114 perror("file_ram_alloc: failed to reinstall signal handler");
1115 exit(1);
1116 }
1117
1118 pthread_sigmask(SIG_SETMASK, &oldset, NULL);
1119 }
1120
1121 block->fd = fd;
1122 return area;
1123
1124 error:
1125 if (mem_prealloc) {
1126 exit(1);
1127 }
1128 return NULL;
1129 }
1130 #else
1131 static void *file_ram_alloc(RAMBlock *block,
1132 ram_addr_t memory,
1133 const char *path)
1134 {
1135 fprintf(stderr, "-mem-path not supported on this host\n");
1136 exit(1);
1137 }
1138 #endif
1139
1140 static ram_addr_t find_ram_offset(ram_addr_t size)
1141 {
1142 RAMBlock *block, *next_block;
1143 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1144
1145 assert(size != 0); /* it would hand out same offset multiple times */
1146
1147 if (QTAILQ_EMPTY(&ram_list.blocks))
1148 return 0;
1149
1150 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1151 ram_addr_t end, next = RAM_ADDR_MAX;
1152
1153 end = block->offset + block->length;
1154
1155 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1156 if (next_block->offset >= end) {
1157 next = MIN(next, next_block->offset);
1158 }
1159 }
1160 if (next - end >= size && next - end < mingap) {
1161 offset = end;
1162 mingap = next - end;
1163 }
1164 }
1165
1166 if (offset == RAM_ADDR_MAX) {
1167 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1168 (uint64_t)size);
1169 abort();
1170 }
1171
1172 return offset;
1173 }
1174
1175 ram_addr_t last_ram_offset(void)
1176 {
1177 RAMBlock *block;
1178 ram_addr_t last = 0;
1179
1180 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1181 last = MAX(last, block->offset + block->length);
1182
1183 return last;
1184 }
1185
1186 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1187 {
1188 int ret;
1189
1190 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1191 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1192 "dump-guest-core", true)) {
1193 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1194 if (ret) {
1195 perror("qemu_madvise");
1196 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1197 "but dump_guest_core=off specified\n");
1198 }
1199 }
1200 }
1201
1202 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1203 {
1204 RAMBlock *new_block, *block;
1205
1206 new_block = NULL;
1207 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1208 if (block->offset == addr) {
1209 new_block = block;
1210 break;
1211 }
1212 }
1213 assert(new_block);
1214 assert(!new_block->idstr[0]);
1215
1216 if (dev) {
1217 char *id = qdev_get_dev_path(dev);
1218 if (id) {
1219 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1220 g_free(id);
1221 }
1222 }
1223 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1224
1225 /* This assumes the iothread lock is taken here too. */
1226 qemu_mutex_lock_ramlist();
1227 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1228 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1229 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1230 new_block->idstr);
1231 abort();
1232 }
1233 }
1234 qemu_mutex_unlock_ramlist();
1235 }
1236
1237 static int memory_try_enable_merging(void *addr, size_t len)
1238 {
1239 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1240 /* disabled by the user */
1241 return 0;
1242 }
1243
1244 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1245 }
1246
1247 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1248 MemoryRegion *mr)
1249 {
1250 RAMBlock *block, *new_block;
1251 ram_addr_t old_ram_size, new_ram_size;
1252
1253 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1254
1255 size = TARGET_PAGE_ALIGN(size);
1256 new_block = g_malloc0(sizeof(*new_block));
1257 new_block->fd = -1;
1258
1259 /* This assumes the iothread lock is taken here too. */
1260 qemu_mutex_lock_ramlist();
1261 new_block->mr = mr;
1262 new_block->offset = find_ram_offset(size);
1263 if (host) {
1264 new_block->host = host;
1265 new_block->flags |= RAM_PREALLOC_MASK;
1266 } else if (xen_enabled()) {
1267 if (mem_path) {
1268 fprintf(stderr, "-mem-path not supported with Xen\n");
1269 exit(1);
1270 }
1271 xen_ram_alloc(new_block->offset, size, mr);
1272 } else {
1273 if (mem_path) {
1274 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1275 /*
1276 * file_ram_alloc() needs to allocate just like
1277 * phys_mem_alloc, but we haven't bothered to provide
1278 * a hook there.
1279 */
1280 fprintf(stderr,
1281 "-mem-path not supported with this accelerator\n");
1282 exit(1);
1283 }
1284 new_block->host = file_ram_alloc(new_block, size, mem_path);
1285 }
1286 if (!new_block->host) {
1287 new_block->host = phys_mem_alloc(size);
1288 if (!new_block->host) {
1289 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1290 new_block->mr->name, strerror(errno));
1291 exit(1);
1292 }
1293 memory_try_enable_merging(new_block->host, size);
1294 }
1295 }
1296 new_block->length = size;
1297
1298 /* Keep the list sorted from biggest to smallest block. */
1299 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1300 if (block->length < new_block->length) {
1301 break;
1302 }
1303 }
1304 if (block) {
1305 QTAILQ_INSERT_BEFORE(block, new_block, next);
1306 } else {
1307 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1308 }
1309 ram_list.mru_block = NULL;
1310
1311 ram_list.version++;
1312 qemu_mutex_unlock_ramlist();
1313
1314 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1315
1316 if (new_ram_size > old_ram_size) {
1317 int i;
1318 for (i = 0; i < DIRTY_MEMORY_NUM; i++) {
1319 ram_list.dirty_memory[i] =
1320 bitmap_zero_extend(ram_list.dirty_memory[i],
1321 old_ram_size, new_ram_size);
1322 }
1323 }
1324 cpu_physical_memory_set_dirty_range(new_block->offset, size);
1325
1326 qemu_ram_setup_dump(new_block->host, size);
1327 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1328 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1329
1330 if (kvm_enabled())
1331 kvm_setup_guest_memory(new_block->host, size);
1332
1333 return new_block->offset;
1334 }
1335
1336 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1337 {
1338 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1339 }
1340
1341 void qemu_ram_free_from_ptr(ram_addr_t addr)
1342 {
1343 RAMBlock *block;
1344
1345 /* This assumes the iothread lock is taken here too. */
1346 qemu_mutex_lock_ramlist();
1347 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1348 if (addr == block->offset) {
1349 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1350 ram_list.mru_block = NULL;
1351 ram_list.version++;
1352 g_free(block);
1353 break;
1354 }
1355 }
1356 qemu_mutex_unlock_ramlist();
1357 }
1358
1359 void qemu_ram_free(ram_addr_t addr)
1360 {
1361 RAMBlock *block;
1362
1363 /* This assumes the iothread lock is taken here too. */
1364 qemu_mutex_lock_ramlist();
1365 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1366 if (addr == block->offset) {
1367 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1368 ram_list.mru_block = NULL;
1369 ram_list.version++;
1370 if (block->flags & RAM_PREALLOC_MASK) {
1371 ;
1372 } else if (xen_enabled()) {
1373 xen_invalidate_map_cache_entry(block->host);
1374 #ifndef _WIN32
1375 } else if (block->fd >= 0) {
1376 munmap(block->host, block->length);
1377 close(block->fd);
1378 #endif
1379 } else {
1380 qemu_anon_ram_free(block->host, block->length);
1381 }
1382 g_free(block);
1383 break;
1384 }
1385 }
1386 qemu_mutex_unlock_ramlist();
1387
1388 }
1389
1390 #ifndef _WIN32
1391 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1392 {
1393 RAMBlock *block;
1394 ram_addr_t offset;
1395 int flags;
1396 void *area, *vaddr;
1397
1398 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1399 offset = addr - block->offset;
1400 if (offset < block->length) {
1401 vaddr = block->host + offset;
1402 if (block->flags & RAM_PREALLOC_MASK) {
1403 ;
1404 } else if (xen_enabled()) {
1405 abort();
1406 } else {
1407 flags = MAP_FIXED;
1408 munmap(vaddr, length);
1409 if (block->fd >= 0) {
1410 #ifdef MAP_POPULATE
1411 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1412 MAP_PRIVATE;
1413 #else
1414 flags |= MAP_PRIVATE;
1415 #endif
1416 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1417 flags, block->fd, offset);
1418 } else {
1419 /*
1420 * Remap needs to match alloc. Accelerators that
1421 * set phys_mem_alloc never remap. If they did,
1422 * we'd need a remap hook here.
1423 */
1424 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1425
1426 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1427 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1428 flags, -1, 0);
1429 }
1430 if (area != vaddr) {
1431 fprintf(stderr, "Could not remap addr: "
1432 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1433 length, addr);
1434 exit(1);
1435 }
1436 memory_try_enable_merging(vaddr, length);
1437 qemu_ram_setup_dump(vaddr, length);
1438 }
1439 return;
1440 }
1441 }
1442 }
1443 #endif /* !_WIN32 */
1444
1445 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1446 With the exception of the softmmu code in this file, this should
1447 only be used for local memory (e.g. video ram) that the device owns,
1448 and knows it isn't going to access beyond the end of the block.
1449
1450 It should not be used for general purpose DMA.
1451 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1452 */
1453 void *qemu_get_ram_ptr(ram_addr_t addr)
1454 {
1455 RAMBlock *block = qemu_get_ram_block(addr);
1456
1457 if (xen_enabled()) {
1458 /* We need to check if the requested address is in the RAM
1459 * because we don't want to map the entire memory in QEMU.
1460 * In that case just map until the end of the page.
1461 */
1462 if (block->offset == 0) {
1463 return xen_map_cache(addr, 0, 0);
1464 } else if (block->host == NULL) {
1465 block->host =
1466 xen_map_cache(block->offset, block->length, 1);
1467 }
1468 }
1469 return block->host + (addr - block->offset);
1470 }
1471
1472 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1473 * but takes a size argument */
1474 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1475 {
1476 if (*size == 0) {
1477 return NULL;
1478 }
1479 if (xen_enabled()) {
1480 return xen_map_cache(addr, *size, 1);
1481 } else {
1482 RAMBlock *block;
1483
1484 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1485 if (addr - block->offset < block->length) {
1486 if (addr - block->offset + *size > block->length)
1487 *size = block->length - addr + block->offset;
1488 return block->host + (addr - block->offset);
1489 }
1490 }
1491
1492 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1493 abort();
1494 }
1495 }
1496
1497 /* Some of the softmmu routines need to translate from a host pointer
1498 (typically a TLB entry) back to a ram offset. */
1499 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1500 {
1501 RAMBlock *block;
1502 uint8_t *host = ptr;
1503
1504 if (xen_enabled()) {
1505 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1506 return qemu_get_ram_block(*ram_addr)->mr;
1507 }
1508
1509 block = ram_list.mru_block;
1510 if (block && block->host && host - block->host < block->length) {
1511 goto found;
1512 }
1513
1514 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1515 /* This case append when the block is not mapped. */
1516 if (block->host == NULL) {
1517 continue;
1518 }
1519 if (host - block->host < block->length) {
1520 goto found;
1521 }
1522 }
1523
1524 return NULL;
1525
1526 found:
1527 *ram_addr = block->offset + (host - block->host);
1528 return block->mr;
1529 }
1530
1531 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1532 uint64_t val, unsigned size)
1533 {
1534 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1535 tb_invalidate_phys_page_fast(ram_addr, size);
1536 }
1537 switch (size) {
1538 case 1:
1539 stb_p(qemu_get_ram_ptr(ram_addr), val);
1540 break;
1541 case 2:
1542 stw_p(qemu_get_ram_ptr(ram_addr), val);
1543 break;
1544 case 4:
1545 stl_p(qemu_get_ram_ptr(ram_addr), val);
1546 break;
1547 default:
1548 abort();
1549 }
1550 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
1551 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
1552 /* we remove the notdirty callback only if the code has been
1553 flushed */
1554 if (!cpu_physical_memory_is_clean(ram_addr)) {
1555 CPUArchState *env = current_cpu->env_ptr;
1556 tlb_set_dirty(env, current_cpu->mem_io_vaddr);
1557 }
1558 }
1559
1560 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1561 unsigned size, bool is_write)
1562 {
1563 return is_write;
1564 }
1565
1566 static const MemoryRegionOps notdirty_mem_ops = {
1567 .write = notdirty_mem_write,
1568 .valid.accepts = notdirty_mem_accepts,
1569 .endianness = DEVICE_NATIVE_ENDIAN,
1570 };
1571
1572 /* Generate a debug exception if a watchpoint has been hit. */
1573 static void check_watchpoint(int offset, int len_mask, int flags)
1574 {
1575 CPUState *cpu = current_cpu;
1576 CPUArchState *env = cpu->env_ptr;
1577 target_ulong pc, cs_base;
1578 target_ulong vaddr;
1579 CPUWatchpoint *wp;
1580 int cpu_flags;
1581
1582 if (env->watchpoint_hit) {
1583 /* We re-entered the check after replacing the TB. Now raise
1584 * the debug interrupt so that is will trigger after the
1585 * current instruction. */
1586 cpu_interrupt(cpu, CPU_INTERRUPT_DEBUG);
1587 return;
1588 }
1589 vaddr = (cpu->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1590 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1591 if ((vaddr == (wp->vaddr & len_mask) ||
1592 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1593 wp->flags |= BP_WATCHPOINT_HIT;
1594 if (!env->watchpoint_hit) {
1595 env->watchpoint_hit = wp;
1596 tb_check_watchpoint(env);
1597 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1598 env->exception_index = EXCP_DEBUG;
1599 cpu_loop_exit(env);
1600 } else {
1601 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1602 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1603 cpu_resume_from_signal(env, NULL);
1604 }
1605 }
1606 } else {
1607 wp->flags &= ~BP_WATCHPOINT_HIT;
1608 }
1609 }
1610 }
1611
1612 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1613 so these check for a hit then pass through to the normal out-of-line
1614 phys routines. */
1615 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1616 unsigned size)
1617 {
1618 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1619 switch (size) {
1620 case 1: return ldub_phys(&address_space_memory, addr);
1621 case 2: return lduw_phys(&address_space_memory, addr);
1622 case 4: return ldl_phys(&address_space_memory, addr);
1623 default: abort();
1624 }
1625 }
1626
1627 static void watch_mem_write(void *opaque, hwaddr addr,
1628 uint64_t val, unsigned size)
1629 {
1630 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1631 switch (size) {
1632 case 1:
1633 stb_phys(&address_space_memory, addr, val);
1634 break;
1635 case 2:
1636 stw_phys(&address_space_memory, addr, val);
1637 break;
1638 case 4:
1639 stl_phys(&address_space_memory, addr, val);
1640 break;
1641 default: abort();
1642 }
1643 }
1644
1645 static const MemoryRegionOps watch_mem_ops = {
1646 .read = watch_mem_read,
1647 .write = watch_mem_write,
1648 .endianness = DEVICE_NATIVE_ENDIAN,
1649 };
1650
1651 static uint64_t subpage_read(void *opaque, hwaddr addr,
1652 unsigned len)
1653 {
1654 subpage_t *subpage = opaque;
1655 uint8_t buf[4];
1656
1657 #if defined(DEBUG_SUBPAGE)
1658 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1659 subpage, len, addr);
1660 #endif
1661 address_space_read(subpage->as, addr + subpage->base, buf, len);
1662 switch (len) {
1663 case 1:
1664 return ldub_p(buf);
1665 case 2:
1666 return lduw_p(buf);
1667 case 4:
1668 return ldl_p(buf);
1669 default:
1670 abort();
1671 }
1672 }
1673
1674 static void subpage_write(void *opaque, hwaddr addr,
1675 uint64_t value, unsigned len)
1676 {
1677 subpage_t *subpage = opaque;
1678 uint8_t buf[4];
1679
1680 #if defined(DEBUG_SUBPAGE)
1681 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1682 " value %"PRIx64"\n",
1683 __func__, subpage, len, addr, value);
1684 #endif
1685 switch (len) {
1686 case 1:
1687 stb_p(buf, value);
1688 break;
1689 case 2:
1690 stw_p(buf, value);
1691 break;
1692 case 4:
1693 stl_p(buf, value);
1694 break;
1695 default:
1696 abort();
1697 }
1698 address_space_write(subpage->as, addr + subpage->base, buf, len);
1699 }
1700
1701 static bool subpage_accepts(void *opaque, hwaddr addr,
1702 unsigned len, bool is_write)
1703 {
1704 subpage_t *subpage = opaque;
1705 #if defined(DEBUG_SUBPAGE)
1706 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1707 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1708 #endif
1709
1710 return address_space_access_valid(subpage->as, addr + subpage->base,
1711 len, is_write);
1712 }
1713
1714 static const MemoryRegionOps subpage_ops = {
1715 .read = subpage_read,
1716 .write = subpage_write,
1717 .valid.accepts = subpage_accepts,
1718 .endianness = DEVICE_NATIVE_ENDIAN,
1719 };
1720
1721 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1722 uint16_t section)
1723 {
1724 int idx, eidx;
1725
1726 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1727 return -1;
1728 idx = SUBPAGE_IDX(start);
1729 eidx = SUBPAGE_IDX(end);
1730 #if defined(DEBUG_SUBPAGE)
1731 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1732 __func__, mmio, start, end, idx, eidx, section);
1733 #endif
1734 for (; idx <= eidx; idx++) {
1735 mmio->sub_section[idx] = section;
1736 }
1737
1738 return 0;
1739 }
1740
1741 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1742 {
1743 subpage_t *mmio;
1744
1745 mmio = g_malloc0(sizeof(subpage_t));
1746
1747 mmio->as = as;
1748 mmio->base = base;
1749 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1750 "subpage", TARGET_PAGE_SIZE);
1751 mmio->iomem.subpage = true;
1752 #if defined(DEBUG_SUBPAGE)
1753 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1754 mmio, base, TARGET_PAGE_SIZE);
1755 #endif
1756 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1757
1758 return mmio;
1759 }
1760
1761 static uint16_t dummy_section(PhysPageMap *map, MemoryRegion *mr)
1762 {
1763 MemoryRegionSection section = {
1764 .address_space = &address_space_memory,
1765 .mr = mr,
1766 .offset_within_address_space = 0,
1767 .offset_within_region = 0,
1768 .size = int128_2_64(),
1769 };
1770
1771 return phys_section_add(map, &section);
1772 }
1773
1774 MemoryRegion *iotlb_to_region(AddressSpace *as, hwaddr index)
1775 {
1776 return as->dispatch->map.sections[index & ~TARGET_PAGE_MASK].mr;
1777 }
1778
1779 static void io_mem_init(void)
1780 {
1781 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1782 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1783 "unassigned", UINT64_MAX);
1784 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1785 "notdirty", UINT64_MAX);
1786 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1787 "watch", UINT64_MAX);
1788 }
1789
1790 static void mem_begin(MemoryListener *listener)
1791 {
1792 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1793 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
1794 uint16_t n;
1795
1796 n = dummy_section(&d->map, &io_mem_unassigned);
1797 assert(n == PHYS_SECTION_UNASSIGNED);
1798 n = dummy_section(&d->map, &io_mem_notdirty);
1799 assert(n == PHYS_SECTION_NOTDIRTY);
1800 n = dummy_section(&d->map, &io_mem_rom);
1801 assert(n == PHYS_SECTION_ROM);
1802 n = dummy_section(&d->map, &io_mem_watch);
1803 assert(n == PHYS_SECTION_WATCH);
1804
1805 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
1806 d->as = as;
1807 as->next_dispatch = d;
1808 }
1809
1810 static void mem_commit(MemoryListener *listener)
1811 {
1812 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1813 AddressSpaceDispatch *cur = as->dispatch;
1814 AddressSpaceDispatch *next = as->next_dispatch;
1815
1816 phys_page_compact_all(next, next->map.nodes_nb);
1817
1818 as->dispatch = next;
1819
1820 if (cur) {
1821 phys_sections_free(&cur->map);
1822 g_free(cur);
1823 }
1824 }
1825
1826 static void tcg_commit(MemoryListener *listener)
1827 {
1828 CPUState *cpu;
1829
1830 /* since each CPU stores ram addresses in its TLB cache, we must
1831 reset the modified entries */
1832 /* XXX: slow ! */
1833 CPU_FOREACH(cpu) {
1834 CPUArchState *env = cpu->env_ptr;
1835
1836 /* FIXME: Disentangle the cpu.h circular files deps so we can
1837 directly get the right CPU from listener. */
1838 if (cpu->tcg_as_listener != listener) {
1839 continue;
1840 }
1841 tlb_flush(env, 1);
1842 }
1843 }
1844
1845 static void core_log_global_start(MemoryListener *listener)
1846 {
1847 cpu_physical_memory_set_dirty_tracking(true);
1848 }
1849
1850 static void core_log_global_stop(MemoryListener *listener)
1851 {
1852 cpu_physical_memory_set_dirty_tracking(false);
1853 }
1854
1855 static MemoryListener core_memory_listener = {
1856 .log_global_start = core_log_global_start,
1857 .log_global_stop = core_log_global_stop,
1858 .priority = 1,
1859 };
1860
1861 void address_space_init_dispatch(AddressSpace *as)
1862 {
1863 as->dispatch = NULL;
1864 as->dispatch_listener = (MemoryListener) {
1865 .begin = mem_begin,
1866 .commit = mem_commit,
1867 .region_add = mem_add,
1868 .region_nop = mem_add,
1869 .priority = 0,
1870 };
1871 memory_listener_register(&as->dispatch_listener, as);
1872 }
1873
1874 void address_space_destroy_dispatch(AddressSpace *as)
1875 {
1876 AddressSpaceDispatch *d = as->dispatch;
1877
1878 memory_listener_unregister(&as->dispatch_listener);
1879 g_free(d);
1880 as->dispatch = NULL;
1881 }
1882
1883 static void memory_map_init(void)
1884 {
1885 system_memory = g_malloc(sizeof(*system_memory));
1886
1887 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
1888 address_space_init(&address_space_memory, system_memory, "memory");
1889
1890 system_io = g_malloc(sizeof(*system_io));
1891 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1892 65536);
1893 address_space_init(&address_space_io, system_io, "I/O");
1894
1895 memory_listener_register(&core_memory_listener, &address_space_memory);
1896 }
1897
1898 MemoryRegion *get_system_memory(void)
1899 {
1900 return system_memory;
1901 }
1902
1903 MemoryRegion *get_system_io(void)
1904 {
1905 return system_io;
1906 }
1907
1908 #endif /* !defined(CONFIG_USER_ONLY) */
1909
1910 /* physical memory access (slow version, mainly for debug) */
1911 #if defined(CONFIG_USER_ONLY)
1912 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1913 uint8_t *buf, int len, int is_write)
1914 {
1915 int l, flags;
1916 target_ulong page;
1917 void * p;
1918
1919 while (len > 0) {
1920 page = addr & TARGET_PAGE_MASK;
1921 l = (page + TARGET_PAGE_SIZE) - addr;
1922 if (l > len)
1923 l = len;
1924 flags = page_get_flags(page);
1925 if (!(flags & PAGE_VALID))
1926 return -1;
1927 if (is_write) {
1928 if (!(flags & PAGE_WRITE))
1929 return -1;
1930 /* XXX: this code should not depend on lock_user */
1931 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1932 return -1;
1933 memcpy(p, buf, l);
1934 unlock_user(p, addr, l);
1935 } else {
1936 if (!(flags & PAGE_READ))
1937 return -1;
1938 /* XXX: this code should not depend on lock_user */
1939 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1940 return -1;
1941 memcpy(buf, p, l);
1942 unlock_user(p, addr, 0);
1943 }
1944 len -= l;
1945 buf += l;
1946 addr += l;
1947 }
1948 return 0;
1949 }
1950
1951 #else
1952
1953 static void invalidate_and_set_dirty(hwaddr addr,
1954 hwaddr length)
1955 {
1956 if (cpu_physical_memory_is_clean(addr)) {
1957 /* invalidate code */
1958 tb_invalidate_phys_page_range(addr, addr + length, 0);
1959 /* set dirty bit */
1960 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_VGA);
1961 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
1962 }
1963 xen_modified_memory(addr, length);
1964 }
1965
1966 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1967 {
1968 unsigned access_size_max = mr->ops->valid.max_access_size;
1969
1970 /* Regions are assumed to support 1-4 byte accesses unless
1971 otherwise specified. */
1972 if (access_size_max == 0) {
1973 access_size_max = 4;
1974 }
1975
1976 /* Bound the maximum access by the alignment of the address. */
1977 if (!mr->ops->impl.unaligned) {
1978 unsigned align_size_max = addr & -addr;
1979 if (align_size_max != 0 && align_size_max < access_size_max) {
1980 access_size_max = align_size_max;
1981 }
1982 }
1983
1984 /* Don't attempt accesses larger than the maximum. */
1985 if (l > access_size_max) {
1986 l = access_size_max;
1987 }
1988 if (l & (l - 1)) {
1989 l = 1 << (qemu_fls(l) - 1);
1990 }
1991
1992 return l;
1993 }
1994
1995 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1996 int len, bool is_write)
1997 {
1998 hwaddr l;
1999 uint8_t *ptr;
2000 uint64_t val;
2001 hwaddr addr1;
2002 MemoryRegion *mr;
2003 bool error = false;
2004
2005 while (len > 0) {
2006 l = len;
2007 mr = address_space_translate(as, addr, &addr1, &l, is_write);
2008
2009 if (is_write) {
2010 if (!memory_access_is_direct(mr, is_write)) {
2011 l = memory_access_size(mr, l, addr1);
2012 /* XXX: could force current_cpu to NULL to avoid
2013 potential bugs */
2014 switch (l) {
2015 case 8:
2016 /* 64 bit write access */
2017 val = ldq_p(buf);
2018 error |= io_mem_write(mr, addr1, val, 8);
2019 break;
2020 case 4:
2021 /* 32 bit write access */
2022 val = ldl_p(buf);
2023 error |= io_mem_write(mr, addr1, val, 4);
2024 break;
2025 case 2:
2026 /* 16 bit write access */
2027 val = lduw_p(buf);
2028 error |= io_mem_write(mr, addr1, val, 2);
2029 break;
2030 case 1:
2031 /* 8 bit write access */
2032 val = ldub_p(buf);
2033 error |= io_mem_write(mr, addr1, val, 1);
2034 break;
2035 default:
2036 abort();
2037 }
2038 } else {
2039 addr1 += memory_region_get_ram_addr(mr);
2040 /* RAM case */
2041 ptr = qemu_get_ram_ptr(addr1);
2042 memcpy(ptr, buf, l);
2043 invalidate_and_set_dirty(addr1, l);
2044 }
2045 } else {
2046 if (!memory_access_is_direct(mr, is_write)) {
2047 /* I/O case */
2048 l = memory_access_size(mr, l, addr1);
2049 switch (l) {
2050 case 8:
2051 /* 64 bit read access */
2052 error |= io_mem_read(mr, addr1, &val, 8);
2053 stq_p(buf, val);
2054 break;
2055 case 4:
2056 /* 32 bit read access */
2057 error |= io_mem_read(mr, addr1, &val, 4);
2058 stl_p(buf, val);
2059 break;
2060 case 2:
2061 /* 16 bit read access */
2062 error |= io_mem_read(mr, addr1, &val, 2);
2063 stw_p(buf, val);
2064 break;
2065 case 1:
2066 /* 8 bit read access */
2067 error |= io_mem_read(mr, addr1, &val, 1);
2068 stb_p(buf, val);
2069 break;
2070 default:
2071 abort();
2072 }
2073 } else {
2074 /* RAM case */
2075 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2076 memcpy(buf, ptr, l);
2077 }
2078 }
2079 len -= l;
2080 buf += l;
2081 addr += l;
2082 }
2083
2084 return error;
2085 }
2086
2087 bool address_space_write(AddressSpace *as, hwaddr addr,
2088 const uint8_t *buf, int len)
2089 {
2090 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2091 }
2092
2093 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2094 {
2095 return address_space_rw(as, addr, buf, len, false);
2096 }
2097
2098
2099 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2100 int len, int is_write)
2101 {
2102 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2103 }
2104
2105 enum write_rom_type {
2106 WRITE_DATA,
2107 FLUSH_CACHE,
2108 };
2109
2110 static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as,
2111 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2112 {
2113 hwaddr l;
2114 uint8_t *ptr;
2115 hwaddr addr1;
2116 MemoryRegion *mr;
2117
2118 while (len > 0) {
2119 l = len;
2120 mr = address_space_translate(as, addr, &addr1, &l, true);
2121
2122 if (!(memory_region_is_ram(mr) ||
2123 memory_region_is_romd(mr))) {
2124 /* do nothing */
2125 } else {
2126 addr1 += memory_region_get_ram_addr(mr);
2127 /* ROM/RAM case */
2128 ptr = qemu_get_ram_ptr(addr1);
2129 switch (type) {
2130 case WRITE_DATA:
2131 memcpy(ptr, buf, l);
2132 invalidate_and_set_dirty(addr1, l);
2133 break;
2134 case FLUSH_CACHE:
2135 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2136 break;
2137 }
2138 }
2139 len -= l;
2140 buf += l;
2141 addr += l;
2142 }
2143 }
2144
2145 /* used for ROM loading : can write in RAM and ROM */
2146 void cpu_physical_memory_write_rom(AddressSpace *as, hwaddr addr,
2147 const uint8_t *buf, int len)
2148 {
2149 cpu_physical_memory_write_rom_internal(as, addr, buf, len, WRITE_DATA);
2150 }
2151
2152 void cpu_flush_icache_range(hwaddr start, int len)
2153 {
2154 /*
2155 * This function should do the same thing as an icache flush that was
2156 * triggered from within the guest. For TCG we are always cache coherent,
2157 * so there is no need to flush anything. For KVM / Xen we need to flush
2158 * the host's instruction cache at least.
2159 */
2160 if (tcg_enabled()) {
2161 return;
2162 }
2163
2164 cpu_physical_memory_write_rom_internal(&address_space_memory,
2165 start, NULL, len, FLUSH_CACHE);
2166 }
2167
2168 typedef struct {
2169 MemoryRegion *mr;
2170 void *buffer;
2171 hwaddr addr;
2172 hwaddr len;
2173 } BounceBuffer;
2174
2175 static BounceBuffer bounce;
2176
2177 typedef struct MapClient {
2178 void *opaque;
2179 void (*callback)(void *opaque);
2180 QLIST_ENTRY(MapClient) link;
2181 } MapClient;
2182
2183 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2184 = QLIST_HEAD_INITIALIZER(map_client_list);
2185
2186 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2187 {
2188 MapClient *client = g_malloc(sizeof(*client));
2189
2190 client->opaque = opaque;
2191 client->callback = callback;
2192 QLIST_INSERT_HEAD(&map_client_list, client, link);
2193 return client;
2194 }
2195
2196 static void cpu_unregister_map_client(void *_client)
2197 {
2198 MapClient *client = (MapClient *)_client;
2199
2200 QLIST_REMOVE(client, link);
2201 g_free(client);
2202 }
2203
2204 static void cpu_notify_map_clients(void)
2205 {
2206 MapClient *client;
2207
2208 while (!QLIST_EMPTY(&map_client_list)) {
2209 client = QLIST_FIRST(&map_client_list);
2210 client->callback(client->opaque);
2211 cpu_unregister_map_client(client);
2212 }
2213 }
2214
2215 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2216 {
2217 MemoryRegion *mr;
2218 hwaddr l, xlat;
2219
2220 while (len > 0) {
2221 l = len;
2222 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2223 if (!memory_access_is_direct(mr, is_write)) {
2224 l = memory_access_size(mr, l, addr);
2225 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2226 return false;
2227 }
2228 }
2229
2230 len -= l;
2231 addr += l;
2232 }
2233 return true;
2234 }
2235
2236 /* Map a physical memory region into a host virtual address.
2237 * May map a subset of the requested range, given by and returned in *plen.
2238 * May return NULL if resources needed to perform the mapping are exhausted.
2239 * Use only for reads OR writes - not for read-modify-write operations.
2240 * Use cpu_register_map_client() to know when retrying the map operation is
2241 * likely to succeed.
2242 */
2243 void *address_space_map(AddressSpace *as,
2244 hwaddr addr,
2245 hwaddr *plen,
2246 bool is_write)
2247 {
2248 hwaddr len = *plen;
2249 hwaddr done = 0;
2250 hwaddr l, xlat, base;
2251 MemoryRegion *mr, *this_mr;
2252 ram_addr_t raddr;
2253
2254 if (len == 0) {
2255 return NULL;
2256 }
2257
2258 l = len;
2259 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2260 if (!memory_access_is_direct(mr, is_write)) {
2261 if (bounce.buffer) {
2262 return NULL;
2263 }
2264 /* Avoid unbounded allocations */
2265 l = MIN(l, TARGET_PAGE_SIZE);
2266 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2267 bounce.addr = addr;
2268 bounce.len = l;
2269
2270 memory_region_ref(mr);
2271 bounce.mr = mr;
2272 if (!is_write) {
2273 address_space_read(as, addr, bounce.buffer, l);
2274 }
2275
2276 *plen = l;
2277 return bounce.buffer;
2278 }
2279
2280 base = xlat;
2281 raddr = memory_region_get_ram_addr(mr);
2282
2283 for (;;) {
2284 len -= l;
2285 addr += l;
2286 done += l;
2287 if (len == 0) {
2288 break;
2289 }
2290
2291 l = len;
2292 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2293 if (this_mr != mr || xlat != base + done) {
2294 break;
2295 }
2296 }
2297
2298 memory_region_ref(mr);
2299 *plen = done;
2300 return qemu_ram_ptr_length(raddr + base, plen);
2301 }
2302
2303 /* Unmaps a memory region previously mapped by address_space_map().
2304 * Will also mark the memory as dirty if is_write == 1. access_len gives
2305 * the amount of memory that was actually read or written by the caller.
2306 */
2307 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2308 int is_write, hwaddr access_len)
2309 {
2310 if (buffer != bounce.buffer) {
2311 MemoryRegion *mr;
2312 ram_addr_t addr1;
2313
2314 mr = qemu_ram_addr_from_host(buffer, &addr1);
2315 assert(mr != NULL);
2316 if (is_write) {
2317 while (access_len) {
2318 unsigned l;
2319 l = TARGET_PAGE_SIZE;
2320 if (l > access_len)
2321 l = access_len;
2322 invalidate_and_set_dirty(addr1, l);
2323 addr1 += l;
2324 access_len -= l;
2325 }
2326 }
2327 if (xen_enabled()) {
2328 xen_invalidate_map_cache_entry(buffer);
2329 }
2330 memory_region_unref(mr);
2331 return;
2332 }
2333 if (is_write) {
2334 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2335 }
2336 qemu_vfree(bounce.buffer);
2337 bounce.buffer = NULL;
2338 memory_region_unref(bounce.mr);
2339 cpu_notify_map_clients();
2340 }
2341
2342 void *cpu_physical_memory_map(hwaddr addr,
2343 hwaddr *plen,
2344 int is_write)
2345 {
2346 return address_space_map(&address_space_memory, addr, plen, is_write);
2347 }
2348
2349 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2350 int is_write, hwaddr access_len)
2351 {
2352 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2353 }
2354
2355 /* warning: addr must be aligned */
2356 static inline uint32_t ldl_phys_internal(AddressSpace *as, hwaddr addr,
2357 enum device_endian endian)
2358 {
2359 uint8_t *ptr;
2360 uint64_t val;
2361 MemoryRegion *mr;
2362 hwaddr l = 4;
2363 hwaddr addr1;
2364
2365 mr = address_space_translate(as, addr, &addr1, &l, false);
2366 if (l < 4 || !memory_access_is_direct(mr, false)) {
2367 /* I/O case */
2368 io_mem_read(mr, addr1, &val, 4);
2369 #if defined(TARGET_WORDS_BIGENDIAN)
2370 if (endian == DEVICE_LITTLE_ENDIAN) {
2371 val = bswap32(val);
2372 }
2373 #else
2374 if (endian == DEVICE_BIG_ENDIAN) {
2375 val = bswap32(val);
2376 }
2377 #endif
2378 } else {
2379 /* RAM case */
2380 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2381 & TARGET_PAGE_MASK)
2382 + addr1);
2383 switch (endian) {
2384 case DEVICE_LITTLE_ENDIAN:
2385 val = ldl_le_p(ptr);
2386 break;
2387 case DEVICE_BIG_ENDIAN:
2388 val = ldl_be_p(ptr);
2389 break;
2390 default:
2391 val = ldl_p(ptr);
2392 break;
2393 }
2394 }
2395 return val;
2396 }
2397
2398 uint32_t ldl_phys(AddressSpace *as, hwaddr addr)
2399 {
2400 return ldl_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2401 }
2402
2403 uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr)
2404 {
2405 return ldl_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2406 }
2407
2408 uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr)
2409 {
2410 return ldl_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2411 }
2412
2413 /* warning: addr must be aligned */
2414 static inline uint64_t ldq_phys_internal(AddressSpace *as, hwaddr addr,
2415 enum device_endian endian)
2416 {
2417 uint8_t *ptr;
2418 uint64_t val;
2419 MemoryRegion *mr;
2420 hwaddr l = 8;
2421 hwaddr addr1;
2422
2423 mr = address_space_translate(as, addr, &addr1, &l,
2424 false);
2425 if (l < 8 || !memory_access_is_direct(mr, false)) {
2426 /* I/O case */
2427 io_mem_read(mr, addr1, &val, 8);
2428 #if defined(TARGET_WORDS_BIGENDIAN)
2429 if (endian == DEVICE_LITTLE_ENDIAN) {
2430 val = bswap64(val);
2431 }
2432 #else
2433 if (endian == DEVICE_BIG_ENDIAN) {
2434 val = bswap64(val);
2435 }
2436 #endif
2437 } else {
2438 /* RAM case */
2439 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2440 & TARGET_PAGE_MASK)
2441 + addr1);
2442 switch (endian) {
2443 case DEVICE_LITTLE_ENDIAN:
2444 val = ldq_le_p(ptr);
2445 break;
2446 case DEVICE_BIG_ENDIAN:
2447 val = ldq_be_p(ptr);
2448 break;
2449 default:
2450 val = ldq_p(ptr);
2451 break;
2452 }
2453 }
2454 return val;
2455 }
2456
2457 uint64_t ldq_phys(AddressSpace *as, hwaddr addr)
2458 {
2459 return ldq_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2460 }
2461
2462 uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr)
2463 {
2464 return ldq_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2465 }
2466
2467 uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr)
2468 {
2469 return ldq_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2470 }
2471
2472 /* XXX: optimize */
2473 uint32_t ldub_phys(AddressSpace *as, hwaddr addr)
2474 {
2475 uint8_t val;
2476 address_space_rw(as, addr, &val, 1, 0);
2477 return val;
2478 }
2479
2480 /* warning: addr must be aligned */
2481 static inline uint32_t lduw_phys_internal(AddressSpace *as, hwaddr addr,
2482 enum device_endian endian)
2483 {
2484 uint8_t *ptr;
2485 uint64_t val;
2486 MemoryRegion *mr;
2487 hwaddr l = 2;
2488 hwaddr addr1;
2489
2490 mr = address_space_translate(as, addr, &addr1, &l,
2491 false);
2492 if (l < 2 || !memory_access_is_direct(mr, false)) {
2493 /* I/O case */
2494 io_mem_read(mr, addr1, &val, 2);
2495 #if defined(TARGET_WORDS_BIGENDIAN)
2496 if (endian == DEVICE_LITTLE_ENDIAN) {
2497 val = bswap16(val);
2498 }
2499 #else
2500 if (endian == DEVICE_BIG_ENDIAN) {
2501 val = bswap16(val);
2502 }
2503 #endif
2504 } else {
2505 /* RAM case */
2506 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2507 & TARGET_PAGE_MASK)
2508 + addr1);
2509 switch (endian) {
2510 case DEVICE_LITTLE_ENDIAN:
2511 val = lduw_le_p(ptr);
2512 break;
2513 case DEVICE_BIG_ENDIAN:
2514 val = lduw_be_p(ptr);
2515 break;
2516 default:
2517 val = lduw_p(ptr);
2518 break;
2519 }
2520 }
2521 return val;
2522 }
2523
2524 uint32_t lduw_phys(AddressSpace *as, hwaddr addr)
2525 {
2526 return lduw_phys_internal(as, addr, DEVICE_NATIVE_ENDIAN);
2527 }
2528
2529 uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr)
2530 {
2531 return lduw_phys_internal(as, addr, DEVICE_LITTLE_ENDIAN);
2532 }
2533
2534 uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr)
2535 {
2536 return lduw_phys_internal(as, addr, DEVICE_BIG_ENDIAN);
2537 }
2538
2539 /* warning: addr must be aligned. The ram page is not masked as dirty
2540 and the code inside is not invalidated. It is useful if the dirty
2541 bits are used to track modified PTEs */
2542 void stl_phys_notdirty(AddressSpace *as, hwaddr addr, uint32_t val)
2543 {
2544 uint8_t *ptr;
2545 MemoryRegion *mr;
2546 hwaddr l = 4;
2547 hwaddr addr1;
2548
2549 mr = address_space_translate(as, addr, &addr1, &l,
2550 true);
2551 if (l < 4 || !memory_access_is_direct(mr, true)) {
2552 io_mem_write(mr, addr1, val, 4);
2553 } else {
2554 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2555 ptr = qemu_get_ram_ptr(addr1);
2556 stl_p(ptr, val);
2557
2558 if (unlikely(in_migration)) {
2559 if (cpu_physical_memory_is_clean(addr1)) {
2560 /* invalidate code */
2561 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2562 /* set dirty bit */
2563 cpu_physical_memory_set_dirty_flag(addr1,
2564 DIRTY_MEMORY_MIGRATION);
2565 cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
2566 }
2567 }
2568 }
2569 }
2570
2571 /* warning: addr must be aligned */
2572 static inline void stl_phys_internal(AddressSpace *as,
2573 hwaddr addr, uint32_t val,
2574 enum device_endian endian)
2575 {
2576 uint8_t *ptr;
2577 MemoryRegion *mr;
2578 hwaddr l = 4;
2579 hwaddr addr1;
2580
2581 mr = address_space_translate(as, addr, &addr1, &l,
2582 true);
2583 if (l < 4 || !memory_access_is_direct(mr, true)) {
2584 #if defined(TARGET_WORDS_BIGENDIAN)
2585 if (endian == DEVICE_LITTLE_ENDIAN) {
2586 val = bswap32(val);
2587 }
2588 #else
2589 if (endian == DEVICE_BIG_ENDIAN) {
2590 val = bswap32(val);
2591 }
2592 #endif
2593 io_mem_write(mr, addr1, val, 4);
2594 } else {
2595 /* RAM case */
2596 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2597 ptr = qemu_get_ram_ptr(addr1);
2598 switch (endian) {
2599 case DEVICE_LITTLE_ENDIAN:
2600 stl_le_p(ptr, val);
2601 break;
2602 case DEVICE_BIG_ENDIAN:
2603 stl_be_p(ptr, val);
2604 break;
2605 default:
2606 stl_p(ptr, val);
2607 break;
2608 }
2609 invalidate_and_set_dirty(addr1, 4);
2610 }
2611 }
2612
2613 void stl_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2614 {
2615 stl_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
2616 }
2617
2618 void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2619 {
2620 stl_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
2621 }
2622
2623 void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2624 {
2625 stl_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
2626 }
2627
2628 /* XXX: optimize */
2629 void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2630 {
2631 uint8_t v = val;
2632 address_space_rw(as, addr, &v, 1, 1);
2633 }
2634
2635 /* warning: addr must be aligned */
2636 static inline void stw_phys_internal(AddressSpace *as,
2637 hwaddr addr, uint32_t val,
2638 enum device_endian endian)
2639 {
2640 uint8_t *ptr;
2641 MemoryRegion *mr;
2642 hwaddr l = 2;
2643 hwaddr addr1;
2644
2645 mr = address_space_translate(as, addr, &addr1, &l, true);
2646 if (l < 2 || !memory_access_is_direct(mr, true)) {
2647 #if defined(TARGET_WORDS_BIGENDIAN)
2648 if (endian == DEVICE_LITTLE_ENDIAN) {
2649 val = bswap16(val);
2650 }
2651 #else
2652 if (endian == DEVICE_BIG_ENDIAN) {
2653 val = bswap16(val);
2654 }
2655 #endif
2656 io_mem_write(mr, addr1, val, 2);
2657 } else {
2658 /* RAM case */
2659 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2660 ptr = qemu_get_ram_ptr(addr1);
2661 switch (endian) {
2662 case DEVICE_LITTLE_ENDIAN:
2663 stw_le_p(ptr, val);
2664 break;
2665 case DEVICE_BIG_ENDIAN:
2666 stw_be_p(ptr, val);
2667 break;
2668 default:
2669 stw_p(ptr, val);
2670 break;
2671 }
2672 invalidate_and_set_dirty(addr1, 2);
2673 }
2674 }
2675
2676 void stw_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2677 {
2678 stw_phys_internal(as, addr, val, DEVICE_NATIVE_ENDIAN);
2679 }
2680
2681 void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2682 {
2683 stw_phys_internal(as, addr, val, DEVICE_LITTLE_ENDIAN);
2684 }
2685
2686 void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val)
2687 {
2688 stw_phys_internal(as, addr, val, DEVICE_BIG_ENDIAN);
2689 }
2690
2691 /* XXX: optimize */
2692 void stq_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2693 {
2694 val = tswap64(val);
2695 address_space_rw(as, addr, (void *) &val, 8, 1);
2696 }
2697
2698 void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2699 {
2700 val = cpu_to_le64(val);
2701 address_space_rw(as, addr, (void *) &val, 8, 1);
2702 }
2703
2704 void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val)
2705 {
2706 val = cpu_to_be64(val);
2707 address_space_rw(as, addr, (void *) &val, 8, 1);
2708 }
2709
2710 /* virtual memory access for debug (includes writing to ROM) */
2711 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2712 uint8_t *buf, int len, int is_write)
2713 {
2714 int l;
2715 hwaddr phys_addr;
2716 target_ulong page;
2717
2718 while (len > 0) {
2719 page = addr & TARGET_PAGE_MASK;
2720 phys_addr = cpu_get_phys_page_debug(cpu, page);
2721 /* if no physical page mapped, return an error */
2722 if (phys_addr == -1)
2723 return -1;
2724 l = (page + TARGET_PAGE_SIZE) - addr;
2725 if (l > len)
2726 l = len;
2727 phys_addr += (addr & ~TARGET_PAGE_MASK);
2728 if (is_write) {
2729 cpu_physical_memory_write_rom(cpu->as, phys_addr, buf, l);
2730 } else {
2731 address_space_rw(cpu->as, phys_addr, buf, l, 0);
2732 }
2733 len -= l;
2734 buf += l;
2735 addr += l;
2736 }
2737 return 0;
2738 }
2739 #endif
2740
2741 #if !defined(CONFIG_USER_ONLY)
2742
2743 /*
2744 * A helper function for the _utterly broken_ virtio device model to find out if
2745 * it's running on a big endian machine. Don't do this at home kids!
2746 */
2747 bool virtio_is_big_endian(void);
2748 bool virtio_is_big_endian(void)
2749 {
2750 #if defined(TARGET_WORDS_BIGENDIAN)
2751 return true;
2752 #else
2753 return false;
2754 #endif
2755 }
2756
2757 #endif
2758
2759 #ifndef CONFIG_USER_ONLY
2760 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2761 {
2762 MemoryRegion*mr;
2763 hwaddr l = 1;
2764
2765 mr = address_space_translate(&address_space_memory,
2766 phys_addr, &phys_addr, &l, false);
2767
2768 return !(memory_region_is_ram(mr) ||
2769 memory_region_is_romd(mr));
2770 }
2771
2772 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2773 {
2774 RAMBlock *block;
2775
2776 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2777 func(block->host, block->offset, block->length, opaque);
2778 }
2779 }
2780 #endif
CRIS target port of Qemu