?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
bitmap: Add bitmap_zero_extend operation
[qemu.git] / exec.c
blobdb369dc42f62ebf73ec36b48447010870900bc60
1 /*
2 * Virtual page mapping
3 *
4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "tcg.h"
30 #include "hw/hw.h"
31 #include "hw/qdev.h"
32 #include "qemu/osdep.h"
33 #include "sysemu/kvm.h"
34 #include "sysemu/sysemu.h"
35 #include "hw/xen/xen.h"
36 #include "qemu/timer.h"
37 #include "qemu/config-file.h"
38 #include "exec/memory.h"
39 #include "sysemu/dma.h"
40 #include "exec/address-spaces.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #else /* !CONFIG_USER_ONLY */
44 #include "sysemu/xen-mapcache.h"
45 #include "trace.h"
46 #endif
47 #include "exec/cpu-all.h"
48
49 #include "exec/cputlb.h"
50 #include "translate-all.h"
51
52 #include "exec/memory-internal.h"
53 #include "qemu/cache-utils.h"
54
55 #include "qemu/range.h"
56
57 //#define DEBUG_SUBPAGE
58
59 #if !defined(CONFIG_USER_ONLY)
60 static int in_migration;
61
62 RAMList ram_list = { .blocks = QTAILQ_HEAD_INITIALIZER(ram_list.blocks) };
63
64 static MemoryRegion *system_memory;
65 static MemoryRegion *system_io;
66
67 AddressSpace address_space_io;
68 AddressSpace address_space_memory;
69
70 MemoryRegion io_mem_rom, io_mem_notdirty;
71 static MemoryRegion io_mem_unassigned;
72
73 #endif
74
75 struct CPUTailQ cpus = QTAILQ_HEAD_INITIALIZER(cpus);
76 /* current CPU in the current thread. It is only valid inside
77 cpu_exec() */
78 DEFINE_TLS(CPUState *, current_cpu);
79 /* 0 = Do not count executed instructions.
80 1 = Precise instruction counting.
81 2 = Adaptive rate instruction counting. */
82 int use_icount;
83
84 #if !defined(CONFIG_USER_ONLY)
85
86 typedef struct PhysPageEntry PhysPageEntry;
87
88 struct PhysPageEntry {
89 /* How many bits skip to next level (in units of L2_SIZE). 0 for a leaf. */
90 uint32_t skip : 6;
91 /* index into phys_sections (!skip) or phys_map_nodes (skip) */
92 uint32_t ptr : 26;
93 };
94
95 #define PHYS_MAP_NODE_NIL (((uint32_t)~0) >> 6)
96
97 /* Size of the L2 (and L3, etc) page tables. */
98 #define ADDR_SPACE_BITS 64
99
100 #define P_L2_BITS 9
101 #define P_L2_SIZE (1 << P_L2_BITS)
102
103 #define P_L2_LEVELS (((ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / P_L2_BITS) + 1)
104
105 typedef PhysPageEntry Node[P_L2_SIZE];
106
107 typedef struct PhysPageMap {
108 unsigned sections_nb;
109 unsigned sections_nb_alloc;
110 unsigned nodes_nb;
111 unsigned nodes_nb_alloc;
112 Node *nodes;
113 MemoryRegionSection *sections;
114 } PhysPageMap;
115
116 struct AddressSpaceDispatch {
117 /* This is a multi-level map on the physical address space.
118 * The bottom level has pointers to MemoryRegionSections.
119 */
120 PhysPageEntry phys_map;
121 PhysPageMap map;
122 AddressSpace *as;
123 };
124
125 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
126 typedef struct subpage_t {
127 MemoryRegion iomem;
128 AddressSpace *as;
129 hwaddr base;
130 uint16_t sub_section[TARGET_PAGE_SIZE];
131 } subpage_t;
132
133 #define PHYS_SECTION_UNASSIGNED 0
134 #define PHYS_SECTION_NOTDIRTY 1
135 #define PHYS_SECTION_ROM 2
136 #define PHYS_SECTION_WATCH 3
137
138 static void io_mem_init(void);
139 static void memory_map_init(void);
140
141 static MemoryRegion io_mem_watch;
142 #endif
143
144 #if !defined(CONFIG_USER_ONLY)
145
146 static void phys_map_node_reserve(PhysPageMap *map, unsigned nodes)
147 {
148 if (map->nodes_nb + nodes > map->nodes_nb_alloc) {
149 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc * 2, 16);
150 map->nodes_nb_alloc = MAX(map->nodes_nb_alloc, map->nodes_nb + nodes);
151 map->nodes = g_renew(Node, map->nodes, map->nodes_nb_alloc);
152 }
153 }
154
155 static uint32_t phys_map_node_alloc(PhysPageMap *map)
156 {
157 unsigned i;
158 uint32_t ret;
159
160 ret = map->nodes_nb++;
161 assert(ret != PHYS_MAP_NODE_NIL);
162 assert(ret != map->nodes_nb_alloc);
163 for (i = 0; i < P_L2_SIZE; ++i) {
164 map->nodes[ret][i].skip = 1;
165 map->nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
166 }
167 return ret;
168 }
169
170 static void phys_page_set_level(PhysPageMap *map, PhysPageEntry *lp,
171 hwaddr *index, hwaddr *nb, uint16_t leaf,
172 int level)
173 {
174 PhysPageEntry *p;
175 int i;
176 hwaddr step = (hwaddr)1 << (level * P_L2_BITS);
177
178 if (lp->skip && lp->ptr == PHYS_MAP_NODE_NIL) {
179 lp->ptr = phys_map_node_alloc(map);
180 p = map->nodes[lp->ptr];
181 if (level == 0) {
182 for (i = 0; i < P_L2_SIZE; i++) {
183 p[i].skip = 0;
184 p[i].ptr = PHYS_SECTION_UNASSIGNED;
185 }
186 }
187 } else {
188 p = map->nodes[lp->ptr];
189 }
190 lp = &p[(*index >> (level * P_L2_BITS)) & (P_L2_SIZE - 1)];
191
192 while (*nb && lp < &p[P_L2_SIZE]) {
193 if ((*index & (step - 1)) == 0 && *nb >= step) {
194 lp->skip = 0;
195 lp->ptr = leaf;
196 *index += step;
197 *nb -= step;
198 } else {
199 phys_page_set_level(map, lp, index, nb, leaf, level - 1);
200 }
201 ++lp;
202 }
203 }
204
205 static void phys_page_set(AddressSpaceDispatch *d,
206 hwaddr index, hwaddr nb,
207 uint16_t leaf)
208 {
209 /* Wildly overreserve - it doesn't matter much. */
210 phys_map_node_reserve(&d->map, 3 * P_L2_LEVELS);
211
212 phys_page_set_level(&d->map, &d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
213 }
214
215 /* Compact a non leaf page entry. Simply detect that the entry has a single child,
216 * and update our entry so we can skip it and go directly to the destination.
217 */
218 static void phys_page_compact(PhysPageEntry *lp, Node *nodes, unsigned long *compacted)
219 {
220 unsigned valid_ptr = P_L2_SIZE;
221 int valid = 0;
222 PhysPageEntry *p;
223 int i;
224
225 if (lp->ptr == PHYS_MAP_NODE_NIL) {
226 return;
227 }
228
229 p = nodes[lp->ptr];
230 for (i = 0; i < P_L2_SIZE; i++) {
231 if (p[i].ptr == PHYS_MAP_NODE_NIL) {
232 continue;
233 }
234
235 valid_ptr = i;
236 valid++;
237 if (p[i].skip) {
238 phys_page_compact(&p[i], nodes, compacted);
239 }
240 }
241
242 /* We can only compress if there's only one child. */
243 if (valid != 1) {
244 return;
245 }
246
247 assert(valid_ptr < P_L2_SIZE);
248
249 /* Don't compress if it won't fit in the # of bits we have. */
250 if (lp->skip + p[valid_ptr].skip >= (1 << 3)) {
251 return;
252 }
253
254 lp->ptr = p[valid_ptr].ptr;
255 if (!p[valid_ptr].skip) {
256 /* If our only child is a leaf, make this a leaf. */
257 /* By design, we should have made this node a leaf to begin with so we
258 * should never reach here.
259 * But since it's so simple to handle this, let's do it just in case we
260 * change this rule.
261 */
262 lp->skip = 0;
263 } else {
264 lp->skip += p[valid_ptr].skip;
265 }
266 }
267
268 static void phys_page_compact_all(AddressSpaceDispatch *d, int nodes_nb)
269 {
270 DECLARE_BITMAP(compacted, nodes_nb);
271
272 if (d->phys_map.skip) {
273 phys_page_compact(&d->phys_map, d->map.nodes, compacted);
274 }
275 }
276
277 static MemoryRegionSection *phys_page_find(PhysPageEntry lp, hwaddr addr,
278 Node *nodes, MemoryRegionSection *sections)
279 {
280 PhysPageEntry *p;
281 hwaddr index = addr >> TARGET_PAGE_BITS;
282 int i;
283
284 for (i = P_L2_LEVELS; lp.skip && (i -= lp.skip) >= 0;) {
285 if (lp.ptr == PHYS_MAP_NODE_NIL) {
286 return &sections[PHYS_SECTION_UNASSIGNED];
287 }
288 p = nodes[lp.ptr];
289 lp = p[(index >> (i * P_L2_BITS)) & (P_L2_SIZE - 1)];
290 }
291
292 if (sections[lp.ptr].size.hi ||
293 range_covers_byte(sections[lp.ptr].offset_within_address_space,
294 sections[lp.ptr].size.lo, addr)) {
295 return &sections[lp.ptr];
296 } else {
297 return &sections[PHYS_SECTION_UNASSIGNED];
298 }
299 }
300
301 bool memory_region_is_unassigned(MemoryRegion *mr)
302 {
303 return mr != &io_mem_rom && mr != &io_mem_notdirty && !mr->rom_device
304 && mr != &io_mem_watch;
305 }
306
307 static MemoryRegionSection *address_space_lookup_region(AddressSpaceDispatch *d,
308 hwaddr addr,
309 bool resolve_subpage)
310 {
311 MemoryRegionSection *section;
312 subpage_t *subpage;
313
314 section = phys_page_find(d->phys_map, addr, d->map.nodes, d->map.sections);
315 if (resolve_subpage && section->mr->subpage) {
316 subpage = container_of(section->mr, subpage_t, iomem);
317 section = &d->map.sections[subpage->sub_section[SUBPAGE_IDX(addr)]];
318 }
319 return section;
320 }
321
322 static MemoryRegionSection *
323 address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *xlat,
324 hwaddr *plen, bool resolve_subpage)
325 {
326 MemoryRegionSection *section;
327 Int128 diff;
328
329 section = address_space_lookup_region(d, addr, resolve_subpage);
330 /* Compute offset within MemoryRegionSection */
331 addr -= section->offset_within_address_space;
332
333 /* Compute offset within MemoryRegion */
334 *xlat = addr + section->offset_within_region;
335
336 diff = int128_sub(section->mr->size, int128_make64(addr));
337 *plen = int128_get64(int128_min(diff, int128_make64(*plen)));
338 return section;
339 }
340
341 MemoryRegion *address_space_translate(AddressSpace *as, hwaddr addr,
342 hwaddr *xlat, hwaddr *plen,
343 bool is_write)
344 {
345 IOMMUTLBEntry iotlb;
346 MemoryRegionSection *section;
347 MemoryRegion *mr;
348 hwaddr len = *plen;
349
350 for (;;) {
351 section = address_space_translate_internal(as->dispatch, addr, &addr, plen, true);
352 mr = section->mr;
353
354 if (!mr->iommu_ops) {
355 break;
356 }
357
358 iotlb = mr->iommu_ops->translate(mr, addr);
359 addr = ((iotlb.translated_addr & ~iotlb.addr_mask)
360 | (addr & iotlb.addr_mask));
361 len = MIN(len, (addr | iotlb.addr_mask) - addr + 1);
362 if (!(iotlb.perm & (1 << is_write))) {
363 mr = &io_mem_unassigned;
364 break;
365 }
366
367 as = iotlb.target_as;
368 }
369
370 *plen = len;
371 *xlat = addr;
372 return mr;
373 }
374
375 MemoryRegionSection *
376 address_space_translate_for_iotlb(AddressSpace *as, hwaddr addr, hwaddr *xlat,
377 hwaddr *plen)
378 {
379 MemoryRegionSection *section;
380 section = address_space_translate_internal(as->dispatch, addr, xlat, plen, false);
381
382 assert(!section->mr->iommu_ops);
383 return section;
384 }
385 #endif
386
387 void cpu_exec_init_all(void)
388 {
389 #if !defined(CONFIG_USER_ONLY)
390 qemu_mutex_init(&ram_list.mutex);
391 memory_map_init();
392 io_mem_init();
393 #endif
394 }
395
396 #if !defined(CONFIG_USER_ONLY)
397
398 static int cpu_common_post_load(void *opaque, int version_id)
399 {
400 CPUState *cpu = opaque;
401
402 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
403 version_id is increased. */
404 cpu->interrupt_request &= ~0x01;
405 tlb_flush(cpu->env_ptr, 1);
406
407 return 0;
408 }
409
410 const VMStateDescription vmstate_cpu_common = {
411 .name = "cpu_common",
412 .version_id = 1,
413 .minimum_version_id = 1,
414 .minimum_version_id_old = 1,
415 .post_load = cpu_common_post_load,
416 .fields = (VMStateField []) {
417 VMSTATE_UINT32(halted, CPUState),
418 VMSTATE_UINT32(interrupt_request, CPUState),
419 VMSTATE_END_OF_LIST()
420 }
421 };
422
423 #endif
424
425 CPUState *qemu_get_cpu(int index)
426 {
427 CPUState *cpu;
428
429 CPU_FOREACH(cpu) {
430 if (cpu->cpu_index == index) {
431 return cpu;
432 }
433 }
434
435 return NULL;
436 }
437
438 void cpu_exec_init(CPUArchState *env)
439 {
440 CPUState *cpu = ENV_GET_CPU(env);
441 CPUClass *cc = CPU_GET_CLASS(cpu);
442 CPUState *some_cpu;
443 int cpu_index;
444
445 #if defined(CONFIG_USER_ONLY)
446 cpu_list_lock();
447 #endif
448 cpu_index = 0;
449 CPU_FOREACH(some_cpu) {
450 cpu_index++;
451 }
452 cpu->cpu_index = cpu_index;
453 cpu->numa_node = 0;
454 QTAILQ_INIT(&env->breakpoints);
455 QTAILQ_INIT(&env->watchpoints);
456 #ifndef CONFIG_USER_ONLY
457 cpu->thread_id = qemu_get_thread_id();
458 #endif
459 QTAILQ_INSERT_TAIL(&cpus, cpu, node);
460 #if defined(CONFIG_USER_ONLY)
461 cpu_list_unlock();
462 #endif
463 if (qdev_get_vmsd(DEVICE(cpu)) == NULL) {
464 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, cpu);
465 }
466 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
467 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
468 cpu_save, cpu_load, env);
469 assert(cc->vmsd == NULL);
470 assert(qdev_get_vmsd(DEVICE(cpu)) == NULL);
471 #endif
472 if (cc->vmsd != NULL) {
473 vmstate_register(NULL, cpu_index, cc->vmsd, cpu);
474 }
475 }
476
477 #if defined(TARGET_HAS_ICE)
478 #if defined(CONFIG_USER_ONLY)
479 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
480 {
481 tb_invalidate_phys_page_range(pc, pc + 1, 0);
482 }
483 #else
484 static void breakpoint_invalidate(CPUState *cpu, target_ulong pc)
485 {
486 hwaddr phys = cpu_get_phys_page_debug(cpu, pc);
487 if (phys != -1) {
488 tb_invalidate_phys_addr(phys | (pc & ~TARGET_PAGE_MASK));
489 }
490 }
491 #endif
492 #endif /* TARGET_HAS_ICE */
493
494 #if defined(CONFIG_USER_ONLY)
495 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
496
497 {
498 }
499
500 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
501 int flags, CPUWatchpoint **watchpoint)
502 {
503 return -ENOSYS;
504 }
505 #else
506 /* Add a watchpoint. */
507 int cpu_watchpoint_insert(CPUArchState *env, target_ulong addr, target_ulong len,
508 int flags, CPUWatchpoint **watchpoint)
509 {
510 target_ulong len_mask = ~(len - 1);
511 CPUWatchpoint *wp;
512
513 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
514 if ((len & (len - 1)) || (addr & ~len_mask) ||
515 len == 0 || len > TARGET_PAGE_SIZE) {
516 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
517 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
518 return -EINVAL;
519 }
520 wp = g_malloc(sizeof(*wp));
521
522 wp->vaddr = addr;
523 wp->len_mask = len_mask;
524 wp->flags = flags;
525
526 /* keep all GDB-injected watchpoints in front */
527 if (flags & BP_GDB)
528 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
529 else
530 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
531
532 tlb_flush_page(env, addr);
533
534 if (watchpoint)
535 *watchpoint = wp;
536 return 0;
537 }
538
539 /* Remove a specific watchpoint. */
540 int cpu_watchpoint_remove(CPUArchState *env, target_ulong addr, target_ulong len,
541 int flags)
542 {
543 target_ulong len_mask = ~(len - 1);
544 CPUWatchpoint *wp;
545
546 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
547 if (addr == wp->vaddr && len_mask == wp->len_mask
548 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
549 cpu_watchpoint_remove_by_ref(env, wp);
550 return 0;
551 }
552 }
553 return -ENOENT;
554 }
555
556 /* Remove a specific watchpoint by reference. */
557 void cpu_watchpoint_remove_by_ref(CPUArchState *env, CPUWatchpoint *watchpoint)
558 {
559 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
560
561 tlb_flush_page(env, watchpoint->vaddr);
562
563 g_free(watchpoint);
564 }
565
566 /* Remove all matching watchpoints. */
567 void cpu_watchpoint_remove_all(CPUArchState *env, int mask)
568 {
569 CPUWatchpoint *wp, *next;
570
571 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
572 if (wp->flags & mask)
573 cpu_watchpoint_remove_by_ref(env, wp);
574 }
575 }
576 #endif
577
578 /* Add a breakpoint. */
579 int cpu_breakpoint_insert(CPUArchState *env, target_ulong pc, int flags,
580 CPUBreakpoint **breakpoint)
581 {
582 #if defined(TARGET_HAS_ICE)
583 CPUBreakpoint *bp;
584
585 bp = g_malloc(sizeof(*bp));
586
587 bp->pc = pc;
588 bp->flags = flags;
589
590 /* keep all GDB-injected breakpoints in front */
591 if (flags & BP_GDB) {
592 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
593 } else {
594 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
595 }
596
597 breakpoint_invalidate(ENV_GET_CPU(env), pc);
598
599 if (breakpoint) {
600 *breakpoint = bp;
601 }
602 return 0;
603 #else
604 return -ENOSYS;
605 #endif
606 }
607
608 /* Remove a specific breakpoint. */
609 int cpu_breakpoint_remove(CPUArchState *env, target_ulong pc, int flags)
610 {
611 #if defined(TARGET_HAS_ICE)
612 CPUBreakpoint *bp;
613
614 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
615 if (bp->pc == pc && bp->flags == flags) {
616 cpu_breakpoint_remove_by_ref(env, bp);
617 return 0;
618 }
619 }
620 return -ENOENT;
621 #else
622 return -ENOSYS;
623 #endif
624 }
625
626 /* Remove a specific breakpoint by reference. */
627 void cpu_breakpoint_remove_by_ref(CPUArchState *env, CPUBreakpoint *breakpoint)
628 {
629 #if defined(TARGET_HAS_ICE)
630 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
631
632 breakpoint_invalidate(ENV_GET_CPU(env), breakpoint->pc);
633
634 g_free(breakpoint);
635 #endif
636 }
637
638 /* Remove all matching breakpoints. */
639 void cpu_breakpoint_remove_all(CPUArchState *env, int mask)
640 {
641 #if defined(TARGET_HAS_ICE)
642 CPUBreakpoint *bp, *next;
643
644 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
645 if (bp->flags & mask)
646 cpu_breakpoint_remove_by_ref(env, bp);
647 }
648 #endif
649 }
650
651 /* enable or disable single step mode. EXCP_DEBUG is returned by the
652 CPU loop after each instruction */
653 void cpu_single_step(CPUState *cpu, int enabled)
654 {
655 #if defined(TARGET_HAS_ICE)
656 if (cpu->singlestep_enabled != enabled) {
657 cpu->singlestep_enabled = enabled;
658 if (kvm_enabled()) {
659 kvm_update_guest_debug(cpu, 0);
660 } else {
661 /* must flush all the translated code to avoid inconsistencies */
662 /* XXX: only flush what is necessary */
663 CPUArchState *env = cpu->env_ptr;
664 tb_flush(env);
665 }
666 }
667 #endif
668 }
669
670 void cpu_abort(CPUArchState *env, const char *fmt, ...)
671 {
672 CPUState *cpu = ENV_GET_CPU(env);
673 va_list ap;
674 va_list ap2;
675
676 va_start(ap, fmt);
677 va_copy(ap2, ap);
678 fprintf(stderr, "qemu: fatal: ");
679 vfprintf(stderr, fmt, ap);
680 fprintf(stderr, "\n");
681 cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
682 if (qemu_log_enabled()) {
683 qemu_log("qemu: fatal: ");
684 qemu_log_vprintf(fmt, ap2);
685 qemu_log("\n");
686 log_cpu_state(cpu, CPU_DUMP_FPU | CPU_DUMP_CCOP);
687 qemu_log_flush();
688 qemu_log_close();
689 }
690 va_end(ap2);
691 va_end(ap);
692 #if defined(CONFIG_USER_ONLY)
693 {
694 struct sigaction act;
695 sigfillset(&act.sa_mask);
696 act.sa_handler = SIG_DFL;
697 sigaction(SIGABRT, &act, NULL);
698 }
699 #endif
700 abort();
701 }
702
703 #if !defined(CONFIG_USER_ONLY)
704 static RAMBlock *qemu_get_ram_block(ram_addr_t addr)
705 {
706 RAMBlock *block;
707
708 /* The list is protected by the iothread lock here. */
709 block = ram_list.mru_block;
710 if (block && addr - block->offset < block->length) {
711 goto found;
712 }
713 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
714 if (addr - block->offset < block->length) {
715 goto found;
716 }
717 }
718
719 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
720 abort();
721
722 found:
723 ram_list.mru_block = block;
724 return block;
725 }
726
727 static void tlb_reset_dirty_range_all(ram_addr_t start, ram_addr_t end,
728 uintptr_t length)
729 {
730 RAMBlock *block;
731 ram_addr_t start1;
732
733 block = qemu_get_ram_block(start);
734 assert(block == qemu_get_ram_block(end - 1));
735 start1 = (uintptr_t)block->host + (start - block->offset);
736 cpu_tlb_reset_dirty_all(start1, length);
737 }
738
739 /* Note: start and end must be within the same ram block. */
740 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
741 unsigned client)
742 {
743 uintptr_t length;
744
745 start &= TARGET_PAGE_MASK;
746 end = TARGET_PAGE_ALIGN(end);
747
748 length = end - start;
749 if (length == 0)
750 return;
751 cpu_physical_memory_mask_dirty_range(start, length, client);
752
753 if (tcg_enabled()) {
754 tlb_reset_dirty_range_all(start, end, length);
755 }
756 }
757
758 static int cpu_physical_memory_set_dirty_tracking(int enable)
759 {
760 int ret = 0;
761 in_migration = enable;
762 return ret;
763 }
764
765 hwaddr memory_region_section_get_iotlb(CPUArchState *env,
766 MemoryRegionSection *section,
767 target_ulong vaddr,
768 hwaddr paddr, hwaddr xlat,
769 int prot,
770 target_ulong *address)
771 {
772 hwaddr iotlb;
773 CPUWatchpoint *wp;
774
775 if (memory_region_is_ram(section->mr)) {
776 /* Normal RAM. */
777 iotlb = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
778 + xlat;
779 if (!section->readonly) {
780 iotlb |= PHYS_SECTION_NOTDIRTY;
781 } else {
782 iotlb |= PHYS_SECTION_ROM;
783 }
784 } else {
785 iotlb = section - address_space_memory.dispatch->map.sections;
786 iotlb += xlat;
787 }
788
789 /* Make accesses to pages with watchpoints go via the
790 watchpoint trap routines. */
791 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
792 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
793 /* Avoid trapping reads of pages with a write breakpoint. */
794 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
795 iotlb = PHYS_SECTION_WATCH + paddr;
796 *address |= TLB_MMIO;
797 break;
798 }
799 }
800 }
801
802 return iotlb;
803 }
804 #endif /* defined(CONFIG_USER_ONLY) */
805
806 #if !defined(CONFIG_USER_ONLY)
807
808 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
809 uint16_t section);
810 static subpage_t *subpage_init(AddressSpace *as, hwaddr base);
811
812 static void *(*phys_mem_alloc)(size_t size) = qemu_anon_ram_alloc;
813
814 /*
815 * Set a custom physical guest memory alloator.
816 * Accelerators with unusual needs may need this. Hopefully, we can
817 * get rid of it eventually.
818 */
819 void phys_mem_set_alloc(void *(*alloc)(size_t))
820 {
821 phys_mem_alloc = alloc;
822 }
823
824 static uint16_t phys_section_add(PhysPageMap *map,
825 MemoryRegionSection *section)
826 {
827 /* The physical section number is ORed with a page-aligned
828 * pointer to produce the iotlb entries. Thus it should
829 * never overflow into the page-aligned value.
830 */
831 assert(map->sections_nb < TARGET_PAGE_SIZE);
832
833 if (map->sections_nb == map->sections_nb_alloc) {
834 map->sections_nb_alloc = MAX(map->sections_nb_alloc * 2, 16);
835 map->sections = g_renew(MemoryRegionSection, map->sections,
836 map->sections_nb_alloc);
837 }
838 map->sections[map->sections_nb] = *section;
839 memory_region_ref(section->mr);
840 return map->sections_nb++;
841 }
842
843 static void phys_section_destroy(MemoryRegion *mr)
844 {
845 memory_region_unref(mr);
846
847 if (mr->subpage) {
848 subpage_t *subpage = container_of(mr, subpage_t, iomem);
849 memory_region_destroy(&subpage->iomem);
850 g_free(subpage);
851 }
852 }
853
854 static void phys_sections_free(PhysPageMap *map)
855 {
856 while (map->sections_nb > 0) {
857 MemoryRegionSection *section = &map->sections[--map->sections_nb];
858 phys_section_destroy(section->mr);
859 }
860 g_free(map->sections);
861 g_free(map->nodes);
862 }
863
864 static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
865 {
866 subpage_t *subpage;
867 hwaddr base = section->offset_within_address_space
868 & TARGET_PAGE_MASK;
869 MemoryRegionSection *existing = phys_page_find(d->phys_map, base,
870 d->map.nodes, d->map.sections);
871 MemoryRegionSection subsection = {
872 .offset_within_address_space = base,
873 .size = int128_make64(TARGET_PAGE_SIZE),
874 };
875 hwaddr start, end;
876
877 assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
878
879 if (!(existing->mr->subpage)) {
880 subpage = subpage_init(d->as, base);
881 subsection.mr = &subpage->iomem;
882 phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
883 phys_section_add(&d->map, &subsection));
884 } else {
885 subpage = container_of(existing->mr, subpage_t, iomem);
886 }
887 start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
888 end = start + int128_get64(section->size) - 1;
889 subpage_register(subpage, start, end,
890 phys_section_add(&d->map, section));
891 }
892
893
894 static void register_multipage(AddressSpaceDispatch *d,
895 MemoryRegionSection *section)
896 {
897 hwaddr start_addr = section->offset_within_address_space;
898 uint16_t section_index = phys_section_add(&d->map, section);
899 uint64_t num_pages = int128_get64(int128_rshift(section->size,
900 TARGET_PAGE_BITS));
901
902 assert(num_pages);
903 phys_page_set(d, start_addr >> TARGET_PAGE_BITS, num_pages, section_index);
904 }
905
906 static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
907 {
908 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
909 AddressSpaceDispatch *d = as->next_dispatch;
910 MemoryRegionSection now = *section, remain = *section;
911 Int128 page_size = int128_make64(TARGET_PAGE_SIZE);
912
913 if (now.offset_within_address_space & ~TARGET_PAGE_MASK) {
914 uint64_t left = TARGET_PAGE_ALIGN(now.offset_within_address_space)
915 - now.offset_within_address_space;
916
917 now.size = int128_min(int128_make64(left), now.size);
918 register_subpage(d, &now);
919 } else {
920 now.size = int128_zero();
921 }
922 while (int128_ne(remain.size, now.size)) {
923 remain.size = int128_sub(remain.size, now.size);
924 remain.offset_within_address_space += int128_get64(now.size);
925 remain.offset_within_region += int128_get64(now.size);
926 now = remain;
927 if (int128_lt(remain.size, page_size)) {
928 register_subpage(d, &now);
929 } else if (remain.offset_within_address_space & ~TARGET_PAGE_MASK) {
930 now.size = page_size;
931 register_subpage(d, &now);
932 } else {
933 now.size = int128_and(now.size, int128_neg(page_size));
934 register_multipage(d, &now);
935 }
936 }
937 }
938
939 void qemu_flush_coalesced_mmio_buffer(void)
940 {
941 if (kvm_enabled())
942 kvm_flush_coalesced_mmio_buffer();
943 }
944
945 void qemu_mutex_lock_ramlist(void)
946 {
947 qemu_mutex_lock(&ram_list.mutex);
948 }
949
950 void qemu_mutex_unlock_ramlist(void)
951 {
952 qemu_mutex_unlock(&ram_list.mutex);
953 }
954
955 #ifdef __linux__
956
957 #include <sys/vfs.h>
958
959 #define HUGETLBFS_MAGIC 0x958458f6
960
961 static long gethugepagesize(const char *path)
962 {
963 struct statfs fs;
964 int ret;
965
966 do {
967 ret = statfs(path, &fs);
968 } while (ret != 0 && errno == EINTR);
969
970 if (ret != 0) {
971 perror(path);
972 return 0;
973 }
974
975 if (fs.f_type != HUGETLBFS_MAGIC)
976 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
977
978 return fs.f_bsize;
979 }
980
981 static sigjmp_buf sigjump;
982
983 static void sigbus_handler(int signal)
984 {
985 siglongjmp(sigjump, 1);
986 }
987
988 static void *file_ram_alloc(RAMBlock *block,
989 ram_addr_t memory,
990 const char *path)
991 {
992 char *filename;
993 char *sanitized_name;
994 char *c;
995 void *area;
996 int fd;
997 unsigned long hpagesize;
998
999 hpagesize = gethugepagesize(path);
1000 if (!hpagesize) {
1001 return NULL;
1002 }
1003
1004 if (memory < hpagesize) {
1005 return NULL;
1006 }
1007
1008 if (kvm_enabled() && !kvm_has_sync_mmu()) {
1009 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
1010 return NULL;
1011 }
1012
1013 /* Make name safe to use with mkstemp by replacing '/' with '_'. */
1014 sanitized_name = g_strdup(block->mr->name);
1015 for (c = sanitized_name; *c != '\0'; c++) {
1016 if (*c == '/')
1017 *c = '_';
1018 }
1019
1020 filename = g_strdup_printf("%s/qemu_back_mem.%s.XXXXXX", path,
1021 sanitized_name);
1022 g_free(sanitized_name);
1023
1024 fd = mkstemp(filename);
1025 if (fd < 0) {
1026 perror("unable to create backing store for hugepages");
1027 g_free(filename);
1028 return NULL;
1029 }
1030 unlink(filename);
1031 g_free(filename);
1032
1033 memory = (memory+hpagesize-1) & ~(hpagesize-1);
1034
1035 /*
1036 * ftruncate is not supported by hugetlbfs in older
1037 * hosts, so don't bother bailing out on errors.
1038 * If anything goes wrong with it under other filesystems,
1039 * mmap will fail.
1040 */
1041 if (ftruncate(fd, memory))
1042 perror("ftruncate");
1043
1044 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
1045 if (area == MAP_FAILED) {
1046 perror("file_ram_alloc: can't mmap RAM pages");
1047 close(fd);
1048 return (NULL);
1049 }
1050
1051 if (mem_prealloc) {
1052 int ret, i;
1053 struct sigaction act, oldact;
1054 sigset_t set, oldset;
1055
1056 memset(&act, 0, sizeof(act));
1057 act.sa_handler = &sigbus_handler;
1058 act.sa_flags = 0;
1059
1060 ret = sigaction(SIGBUS, &act, &oldact);
1061 if (ret) {
1062 perror("file_ram_alloc: failed to install signal handler");
1063 exit(1);
1064 }
1065
1066 /* unblock SIGBUS */
1067 sigemptyset(&set);
1068 sigaddset(&set, SIGBUS);
1069 pthread_sigmask(SIG_UNBLOCK, &set, &oldset);
1070
1071 if (sigsetjmp(sigjump, 1)) {
1072 fprintf(stderr, "file_ram_alloc: failed to preallocate pages\n");
1073 exit(1);
1074 }
1075
1076 /* MAP_POPULATE silently ignores failures */
1077 for (i = 0; i < (memory/hpagesize)-1; i++) {
1078 memset(area + (hpagesize*i), 0, 1);
1079 }
1080
1081 ret = sigaction(SIGBUS, &oldact, NULL);
1082 if (ret) {
1083 perror("file_ram_alloc: failed to reinstall signal handler");
1084 exit(1);
1085 }
1086
1087 pthread_sigmask(SIG_SETMASK, &oldset, NULL);
1088 }
1089
1090 block->fd = fd;
1091 return area;
1092 }
1093 #else
1094 static void *file_ram_alloc(RAMBlock *block,
1095 ram_addr_t memory,
1096 const char *path)
1097 {
1098 fprintf(stderr, "-mem-path not supported on this host\n");
1099 exit(1);
1100 }
1101 #endif
1102
1103 static ram_addr_t find_ram_offset(ram_addr_t size)
1104 {
1105 RAMBlock *block, *next_block;
1106 ram_addr_t offset = RAM_ADDR_MAX, mingap = RAM_ADDR_MAX;
1107
1108 assert(size != 0); /* it would hand out same offset multiple times */
1109
1110 if (QTAILQ_EMPTY(&ram_list.blocks))
1111 return 0;
1112
1113 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1114 ram_addr_t end, next = RAM_ADDR_MAX;
1115
1116 end = block->offset + block->length;
1117
1118 QTAILQ_FOREACH(next_block, &ram_list.blocks, next) {
1119 if (next_block->offset >= end) {
1120 next = MIN(next, next_block->offset);
1121 }
1122 }
1123 if (next - end >= size && next - end < mingap) {
1124 offset = end;
1125 mingap = next - end;
1126 }
1127 }
1128
1129 if (offset == RAM_ADDR_MAX) {
1130 fprintf(stderr, "Failed to find gap of requested size: %" PRIu64 "\n",
1131 (uint64_t)size);
1132 abort();
1133 }
1134
1135 return offset;
1136 }
1137
1138 ram_addr_t last_ram_offset(void)
1139 {
1140 RAMBlock *block;
1141 ram_addr_t last = 0;
1142
1143 QTAILQ_FOREACH(block, &ram_list.blocks, next)
1144 last = MAX(last, block->offset + block->length);
1145
1146 return last;
1147 }
1148
1149 static void qemu_ram_setup_dump(void *addr, ram_addr_t size)
1150 {
1151 int ret;
1152
1153 /* Use MADV_DONTDUMP, if user doesn't want the guest memory in the core */
1154 if (!qemu_opt_get_bool(qemu_get_machine_opts(),
1155 "dump-guest-core", true)) {
1156 ret = qemu_madvise(addr, size, QEMU_MADV_DONTDUMP);
1157 if (ret) {
1158 perror("qemu_madvise");
1159 fprintf(stderr, "madvise doesn't support MADV_DONTDUMP, "
1160 "but dump_guest_core=off specified\n");
1161 }
1162 }
1163 }
1164
1165 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
1166 {
1167 RAMBlock *new_block, *block;
1168
1169 new_block = NULL;
1170 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1171 if (block->offset == addr) {
1172 new_block = block;
1173 break;
1174 }
1175 }
1176 assert(new_block);
1177 assert(!new_block->idstr[0]);
1178
1179 if (dev) {
1180 char *id = qdev_get_dev_path(dev);
1181 if (id) {
1182 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
1183 g_free(id);
1184 }
1185 }
1186 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
1187
1188 /* This assumes the iothread lock is taken here too. */
1189 qemu_mutex_lock_ramlist();
1190 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1191 if (block != new_block && !strcmp(block->idstr, new_block->idstr)) {
1192 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
1193 new_block->idstr);
1194 abort();
1195 }
1196 }
1197 qemu_mutex_unlock_ramlist();
1198 }
1199
1200 static int memory_try_enable_merging(void *addr, size_t len)
1201 {
1202 if (!qemu_opt_get_bool(qemu_get_machine_opts(), "mem-merge", true)) {
1203 /* disabled by the user */
1204 return 0;
1205 }
1206
1207 return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
1208 }
1209
1210 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
1211 MemoryRegion *mr)
1212 {
1213 RAMBlock *block, *new_block;
1214 ram_addr_t old_ram_size, new_ram_size;
1215
1216 old_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1217
1218 size = TARGET_PAGE_ALIGN(size);
1219 new_block = g_malloc0(sizeof(*new_block));
1220 new_block->fd = -1;
1221
1222 /* This assumes the iothread lock is taken here too. */
1223 qemu_mutex_lock_ramlist();
1224 new_block->mr = mr;
1225 new_block->offset = find_ram_offset(size);
1226 if (host) {
1227 new_block->host = host;
1228 new_block->flags |= RAM_PREALLOC_MASK;
1229 } else if (xen_enabled()) {
1230 if (mem_path) {
1231 fprintf(stderr, "-mem-path not supported with Xen\n");
1232 exit(1);
1233 }
1234 xen_ram_alloc(new_block->offset, size, mr);
1235 } else {
1236 if (mem_path) {
1237 if (phys_mem_alloc != qemu_anon_ram_alloc) {
1238 /*
1239 * file_ram_alloc() needs to allocate just like
1240 * phys_mem_alloc, but we haven't bothered to provide
1241 * a hook there.
1242 */
1243 fprintf(stderr,
1244 "-mem-path not supported with this accelerator\n");
1245 exit(1);
1246 }
1247 new_block->host = file_ram_alloc(new_block, size, mem_path);
1248 }
1249 if (!new_block->host) {
1250 new_block->host = phys_mem_alloc(size);
1251 if (!new_block->host) {
1252 fprintf(stderr, "Cannot set up guest memory '%s': %s\n",
1253 new_block->mr->name, strerror(errno));
1254 exit(1);
1255 }
1256 memory_try_enable_merging(new_block->host, size);
1257 }
1258 }
1259 new_block->length = size;
1260
1261 /* Keep the list sorted from biggest to smallest block. */
1262 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1263 if (block->length < new_block->length) {
1264 break;
1265 }
1266 }
1267 if (block) {
1268 QTAILQ_INSERT_BEFORE(block, new_block, next);
1269 } else {
1270 QTAILQ_INSERT_TAIL(&ram_list.blocks, new_block, next);
1271 }
1272 ram_list.mru_block = NULL;
1273
1274 ram_list.version++;
1275 qemu_mutex_unlock_ramlist();
1276
1277 new_ram_size = last_ram_offset() >> TARGET_PAGE_BITS;
1278
1279 if (new_ram_size > old_ram_size) {
1280 ram_list.phys_dirty = g_realloc(ram_list.phys_dirty, new_ram_size);
1281 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
1282 0, size >> TARGET_PAGE_BITS);
1283 }
1284 cpu_physical_memory_set_dirty_range(new_block->offset, size);
1285
1286 qemu_ram_setup_dump(new_block->host, size);
1287 qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
1288 qemu_madvise(new_block->host, size, QEMU_MADV_DONTFORK);
1289
1290 if (kvm_enabled())
1291 kvm_setup_guest_memory(new_block->host, size);
1292
1293 return new_block->offset;
1294 }
1295
1296 ram_addr_t qemu_ram_alloc(ram_addr_t size, MemoryRegion *mr)
1297 {
1298 return qemu_ram_alloc_from_ptr(size, NULL, mr);
1299 }
1300
1301 void qemu_ram_free_from_ptr(ram_addr_t addr)
1302 {
1303 RAMBlock *block;
1304
1305 /* This assumes the iothread lock is taken here too. */
1306 qemu_mutex_lock_ramlist();
1307 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1308 if (addr == block->offset) {
1309 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1310 ram_list.mru_block = NULL;
1311 ram_list.version++;
1312 g_free(block);
1313 break;
1314 }
1315 }
1316 qemu_mutex_unlock_ramlist();
1317 }
1318
1319 void qemu_ram_free(ram_addr_t addr)
1320 {
1321 RAMBlock *block;
1322
1323 /* This assumes the iothread lock is taken here too. */
1324 qemu_mutex_lock_ramlist();
1325 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1326 if (addr == block->offset) {
1327 QTAILQ_REMOVE(&ram_list.blocks, block, next);
1328 ram_list.mru_block = NULL;
1329 ram_list.version++;
1330 if (block->flags & RAM_PREALLOC_MASK) {
1331 ;
1332 } else if (xen_enabled()) {
1333 xen_invalidate_map_cache_entry(block->host);
1334 #ifndef _WIN32
1335 } else if (block->fd >= 0) {
1336 munmap(block->host, block->length);
1337 close(block->fd);
1338 #endif
1339 } else {
1340 qemu_anon_ram_free(block->host, block->length);
1341 }
1342 g_free(block);
1343 break;
1344 }
1345 }
1346 qemu_mutex_unlock_ramlist();
1347
1348 }
1349
1350 #ifndef _WIN32
1351 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
1352 {
1353 RAMBlock *block;
1354 ram_addr_t offset;
1355 int flags;
1356 void *area, *vaddr;
1357
1358 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1359 offset = addr - block->offset;
1360 if (offset < block->length) {
1361 vaddr = block->host + offset;
1362 if (block->flags & RAM_PREALLOC_MASK) {
1363 ;
1364 } else if (xen_enabled()) {
1365 abort();
1366 } else {
1367 flags = MAP_FIXED;
1368 munmap(vaddr, length);
1369 if (block->fd >= 0) {
1370 #ifdef MAP_POPULATE
1371 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
1372 MAP_PRIVATE;
1373 #else
1374 flags |= MAP_PRIVATE;
1375 #endif
1376 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1377 flags, block->fd, offset);
1378 } else {
1379 /*
1380 * Remap needs to match alloc. Accelerators that
1381 * set phys_mem_alloc never remap. If they did,
1382 * we'd need a remap hook here.
1383 */
1384 assert(phys_mem_alloc == qemu_anon_ram_alloc);
1385
1386 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
1387 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
1388 flags, -1, 0);
1389 }
1390 if (area != vaddr) {
1391 fprintf(stderr, "Could not remap addr: "
1392 RAM_ADDR_FMT "@" RAM_ADDR_FMT "\n",
1393 length, addr);
1394 exit(1);
1395 }
1396 memory_try_enable_merging(vaddr, length);
1397 qemu_ram_setup_dump(vaddr, length);
1398 }
1399 return;
1400 }
1401 }
1402 }
1403 #endif /* !_WIN32 */
1404
1405 /* Return a host pointer to ram allocated with qemu_ram_alloc.
1406 With the exception of the softmmu code in this file, this should
1407 only be used for local memory (e.g. video ram) that the device owns,
1408 and knows it isn't going to access beyond the end of the block.
1409
1410 It should not be used for general purpose DMA.
1411 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
1412 */
1413 void *qemu_get_ram_ptr(ram_addr_t addr)
1414 {
1415 RAMBlock *block = qemu_get_ram_block(addr);
1416
1417 if (xen_enabled()) {
1418 /* We need to check if the requested address is in the RAM
1419 * because we don't want to map the entire memory in QEMU.
1420 * In that case just map until the end of the page.
1421 */
1422 if (block->offset == 0) {
1423 return xen_map_cache(addr, 0, 0);
1424 } else if (block->host == NULL) {
1425 block->host =
1426 xen_map_cache(block->offset, block->length, 1);
1427 }
1428 }
1429 return block->host + (addr - block->offset);
1430 }
1431
1432 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
1433 * but takes a size argument */
1434 static void *qemu_ram_ptr_length(ram_addr_t addr, hwaddr *size)
1435 {
1436 if (*size == 0) {
1437 return NULL;
1438 }
1439 if (xen_enabled()) {
1440 return xen_map_cache(addr, *size, 1);
1441 } else {
1442 RAMBlock *block;
1443
1444 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1445 if (addr - block->offset < block->length) {
1446 if (addr - block->offset + *size > block->length)
1447 *size = block->length - addr + block->offset;
1448 return block->host + (addr - block->offset);
1449 }
1450 }
1451
1452 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
1453 abort();
1454 }
1455 }
1456
1457 /* Some of the softmmu routines need to translate from a host pointer
1458 (typically a TLB entry) back to a ram offset. */
1459 MemoryRegion *qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
1460 {
1461 RAMBlock *block;
1462 uint8_t *host = ptr;
1463
1464 if (xen_enabled()) {
1465 *ram_addr = xen_ram_addr_from_mapcache(ptr);
1466 return qemu_get_ram_block(*ram_addr)->mr;
1467 }
1468
1469 block = ram_list.mru_block;
1470 if (block && block->host && host - block->host < block->length) {
1471 goto found;
1472 }
1473
1474 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
1475 /* This case append when the block is not mapped. */
1476 if (block->host == NULL) {
1477 continue;
1478 }
1479 if (host - block->host < block->length) {
1480 goto found;
1481 }
1482 }
1483
1484 return NULL;
1485
1486 found:
1487 *ram_addr = block->offset + (host - block->host);
1488 return block->mr;
1489 }
1490
1491 static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
1492 uint64_t val, unsigned size)
1493 {
1494 if (!cpu_physical_memory_get_dirty_flag(ram_addr, DIRTY_MEMORY_CODE)) {
1495 tb_invalidate_phys_page_fast(ram_addr, size);
1496 }
1497 switch (size) {
1498 case 1:
1499 stb_p(qemu_get_ram_ptr(ram_addr), val);
1500 break;
1501 case 2:
1502 stw_p(qemu_get_ram_ptr(ram_addr), val);
1503 break;
1504 case 4:
1505 stl_p(qemu_get_ram_ptr(ram_addr), val);
1506 break;
1507 default:
1508 abort();
1509 }
1510 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_MIGRATION);
1511 cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_VGA);
1512 /* we remove the notdirty callback only if the code has been
1513 flushed */
1514 if (cpu_physical_memory_is_dirty(ram_addr)) {
1515 CPUArchState *env = current_cpu->env_ptr;
1516 tlb_set_dirty(env, env->mem_io_vaddr);
1517 }
1518 }
1519
1520 static bool notdirty_mem_accepts(void *opaque, hwaddr addr,
1521 unsigned size, bool is_write)
1522 {
1523 return is_write;
1524 }
1525
1526 static const MemoryRegionOps notdirty_mem_ops = {
1527 .write = notdirty_mem_write,
1528 .valid.accepts = notdirty_mem_accepts,
1529 .endianness = DEVICE_NATIVE_ENDIAN,
1530 };
1531
1532 /* Generate a debug exception if a watchpoint has been hit. */
1533 static void check_watchpoint(int offset, int len_mask, int flags)
1534 {
1535 CPUArchState *env = current_cpu->env_ptr;
1536 target_ulong pc, cs_base;
1537 target_ulong vaddr;
1538 CPUWatchpoint *wp;
1539 int cpu_flags;
1540
1541 if (env->watchpoint_hit) {
1542 /* We re-entered the check after replacing the TB. Now raise
1543 * the debug interrupt so that is will trigger after the
1544 * current instruction. */
1545 cpu_interrupt(ENV_GET_CPU(env), CPU_INTERRUPT_DEBUG);
1546 return;
1547 }
1548 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
1549 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1550 if ((vaddr == (wp->vaddr & len_mask) ||
1551 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
1552 wp->flags |= BP_WATCHPOINT_HIT;
1553 if (!env->watchpoint_hit) {
1554 env->watchpoint_hit = wp;
1555 tb_check_watchpoint(env);
1556 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
1557 env->exception_index = EXCP_DEBUG;
1558 cpu_loop_exit(env);
1559 } else {
1560 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
1561 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
1562 cpu_resume_from_signal(env, NULL);
1563 }
1564 }
1565 } else {
1566 wp->flags &= ~BP_WATCHPOINT_HIT;
1567 }
1568 }
1569 }
1570
1571 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
1572 so these check for a hit then pass through to the normal out-of-line
1573 phys routines. */
1574 static uint64_t watch_mem_read(void *opaque, hwaddr addr,
1575 unsigned size)
1576 {
1577 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
1578 switch (size) {
1579 case 1: return ldub_phys(addr);
1580 case 2: return lduw_phys(addr);
1581 case 4: return ldl_phys(addr);
1582 default: abort();
1583 }
1584 }
1585
1586 static void watch_mem_write(void *opaque, hwaddr addr,
1587 uint64_t val, unsigned size)
1588 {
1589 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
1590 switch (size) {
1591 case 1:
1592 stb_phys(addr, val);
1593 break;
1594 case 2:
1595 stw_phys(addr, val);
1596 break;
1597 case 4:
1598 stl_phys(addr, val);
1599 break;
1600 default: abort();
1601 }
1602 }
1603
1604 static const MemoryRegionOps watch_mem_ops = {
1605 .read = watch_mem_read,
1606 .write = watch_mem_write,
1607 .endianness = DEVICE_NATIVE_ENDIAN,
1608 };
1609
1610 static uint64_t subpage_read(void *opaque, hwaddr addr,
1611 unsigned len)
1612 {
1613 subpage_t *subpage = opaque;
1614 uint8_t buf[4];
1615
1616 #if defined(DEBUG_SUBPAGE)
1617 printf("%s: subpage %p len %u addr " TARGET_FMT_plx "\n", __func__,
1618 subpage, len, addr);
1619 #endif
1620 address_space_read(subpage->as, addr + subpage->base, buf, len);
1621 switch (len) {
1622 case 1:
1623 return ldub_p(buf);
1624 case 2:
1625 return lduw_p(buf);
1626 case 4:
1627 return ldl_p(buf);
1628 default:
1629 abort();
1630 }
1631 }
1632
1633 static void subpage_write(void *opaque, hwaddr addr,
1634 uint64_t value, unsigned len)
1635 {
1636 subpage_t *subpage = opaque;
1637 uint8_t buf[4];
1638
1639 #if defined(DEBUG_SUBPAGE)
1640 printf("%s: subpage %p len %u addr " TARGET_FMT_plx
1641 " value %"PRIx64"\n",
1642 __func__, subpage, len, addr, value);
1643 #endif
1644 switch (len) {
1645 case 1:
1646 stb_p(buf, value);
1647 break;
1648 case 2:
1649 stw_p(buf, value);
1650 break;
1651 case 4:
1652 stl_p(buf, value);
1653 break;
1654 default:
1655 abort();
1656 }
1657 address_space_write(subpage->as, addr + subpage->base, buf, len);
1658 }
1659
1660 static bool subpage_accepts(void *opaque, hwaddr addr,
1661 unsigned len, bool is_write)
1662 {
1663 subpage_t *subpage = opaque;
1664 #if defined(DEBUG_SUBPAGE)
1665 printf("%s: subpage %p %c len %u addr " TARGET_FMT_plx "\n",
1666 __func__, subpage, is_write ? 'w' : 'r', len, addr);
1667 #endif
1668
1669 return address_space_access_valid(subpage->as, addr + subpage->base,
1670 len, is_write);
1671 }
1672
1673 static const MemoryRegionOps subpage_ops = {
1674 .read = subpage_read,
1675 .write = subpage_write,
1676 .valid.accepts = subpage_accepts,
1677 .endianness = DEVICE_NATIVE_ENDIAN,
1678 };
1679
1680 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
1681 uint16_t section)
1682 {
1683 int idx, eidx;
1684
1685 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
1686 return -1;
1687 idx = SUBPAGE_IDX(start);
1688 eidx = SUBPAGE_IDX(end);
1689 #if defined(DEBUG_SUBPAGE)
1690 printf("%s: %p start %08x end %08x idx %08x eidx %08x section %d\n",
1691 __func__, mmio, start, end, idx, eidx, section);
1692 #endif
1693 for (; idx <= eidx; idx++) {
1694 mmio->sub_section[idx] = section;
1695 }
1696
1697 return 0;
1698 }
1699
1700 static subpage_t *subpage_init(AddressSpace *as, hwaddr base)
1701 {
1702 subpage_t *mmio;
1703
1704 mmio = g_malloc0(sizeof(subpage_t));
1705
1706 mmio->as = as;
1707 mmio->base = base;
1708 memory_region_init_io(&mmio->iomem, NULL, &subpage_ops, mmio,
1709 "subpage", TARGET_PAGE_SIZE);
1710 mmio->iomem.subpage = true;
1711 #if defined(DEBUG_SUBPAGE)
1712 printf("%s: %p base " TARGET_FMT_plx " len %08x\n", __func__,
1713 mmio, base, TARGET_PAGE_SIZE);
1714 #endif
1715 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, PHYS_SECTION_UNASSIGNED);
1716
1717 return mmio;
1718 }
1719
1720 static uint16_t dummy_section(PhysPageMap *map, MemoryRegion *mr)
1721 {
1722 MemoryRegionSection section = {
1723 .mr = mr,
1724 .offset_within_address_space = 0,
1725 .offset_within_region = 0,
1726 .size = int128_2_64(),
1727 };
1728
1729 return phys_section_add(map, &section);
1730 }
1731
1732 MemoryRegion *iotlb_to_region(hwaddr index)
1733 {
1734 return address_space_memory.dispatch->map.sections[
1735 index & ~TARGET_PAGE_MASK].mr;
1736 }
1737
1738 static void io_mem_init(void)
1739 {
1740 memory_region_init_io(&io_mem_rom, NULL, &unassigned_mem_ops, NULL, "rom", UINT64_MAX);
1741 memory_region_init_io(&io_mem_unassigned, NULL, &unassigned_mem_ops, NULL,
1742 "unassigned", UINT64_MAX);
1743 memory_region_init_io(&io_mem_notdirty, NULL, &notdirty_mem_ops, NULL,
1744 "notdirty", UINT64_MAX);
1745 memory_region_init_io(&io_mem_watch, NULL, &watch_mem_ops, NULL,
1746 "watch", UINT64_MAX);
1747 }
1748
1749 static void mem_begin(MemoryListener *listener)
1750 {
1751 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1752 AddressSpaceDispatch *d = g_new0(AddressSpaceDispatch, 1);
1753 uint16_t n;
1754
1755 n = dummy_section(&d->map, &io_mem_unassigned);
1756 assert(n == PHYS_SECTION_UNASSIGNED);
1757 n = dummy_section(&d->map, &io_mem_notdirty);
1758 assert(n == PHYS_SECTION_NOTDIRTY);
1759 n = dummy_section(&d->map, &io_mem_rom);
1760 assert(n == PHYS_SECTION_ROM);
1761 n = dummy_section(&d->map, &io_mem_watch);
1762 assert(n == PHYS_SECTION_WATCH);
1763
1764 d->phys_map = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .skip = 1 };
1765 d->as = as;
1766 as->next_dispatch = d;
1767 }
1768
1769 static void mem_commit(MemoryListener *listener)
1770 {
1771 AddressSpace *as = container_of(listener, AddressSpace, dispatch_listener);
1772 AddressSpaceDispatch *cur = as->dispatch;
1773 AddressSpaceDispatch *next = as->next_dispatch;
1774
1775 phys_page_compact_all(next, next->map.nodes_nb);
1776
1777 as->dispatch = next;
1778
1779 if (cur) {
1780 phys_sections_free(&cur->map);
1781 g_free(cur);
1782 }
1783 }
1784
1785 static void tcg_commit(MemoryListener *listener)
1786 {
1787 CPUState *cpu;
1788
1789 /* since each CPU stores ram addresses in its TLB cache, we must
1790 reset the modified entries */
1791 /* XXX: slow ! */
1792 CPU_FOREACH(cpu) {
1793 CPUArchState *env = cpu->env_ptr;
1794
1795 tlb_flush(env, 1);
1796 }
1797 }
1798
1799 static void core_log_global_start(MemoryListener *listener)
1800 {
1801 cpu_physical_memory_set_dirty_tracking(1);
1802 }
1803
1804 static void core_log_global_stop(MemoryListener *listener)
1805 {
1806 cpu_physical_memory_set_dirty_tracking(0);
1807 }
1808
1809 static MemoryListener core_memory_listener = {
1810 .log_global_start = core_log_global_start,
1811 .log_global_stop = core_log_global_stop,
1812 .priority = 1,
1813 };
1814
1815 static MemoryListener tcg_memory_listener = {
1816 .commit = tcg_commit,
1817 };
1818
1819 void address_space_init_dispatch(AddressSpace *as)
1820 {
1821 as->dispatch = NULL;
1822 as->dispatch_listener = (MemoryListener) {
1823 .begin = mem_begin,
1824 .commit = mem_commit,
1825 .region_add = mem_add,
1826 .region_nop = mem_add,
1827 .priority = 0,
1828 };
1829 memory_listener_register(&as->dispatch_listener, as);
1830 }
1831
1832 void address_space_destroy_dispatch(AddressSpace *as)
1833 {
1834 AddressSpaceDispatch *d = as->dispatch;
1835
1836 memory_listener_unregister(&as->dispatch_listener);
1837 g_free(d);
1838 as->dispatch = NULL;
1839 }
1840
1841 static void memory_map_init(void)
1842 {
1843 system_memory = g_malloc(sizeof(*system_memory));
1844
1845 memory_region_init(system_memory, NULL, "system", UINT64_MAX);
1846 address_space_init(&address_space_memory, system_memory, "memory");
1847
1848 system_io = g_malloc(sizeof(*system_io));
1849 memory_region_init_io(system_io, NULL, &unassigned_io_ops, NULL, "io",
1850 65536);
1851 address_space_init(&address_space_io, system_io, "I/O");
1852
1853 memory_listener_register(&core_memory_listener, &address_space_memory);
1854 if (tcg_enabled()) {
1855 memory_listener_register(&tcg_memory_listener, &address_space_memory);
1856 }
1857 }
1858
1859 MemoryRegion *get_system_memory(void)
1860 {
1861 return system_memory;
1862 }
1863
1864 MemoryRegion *get_system_io(void)
1865 {
1866 return system_io;
1867 }
1868
1869 #endif /* !defined(CONFIG_USER_ONLY) */
1870
1871 /* physical memory access (slow version, mainly for debug) */
1872 #if defined(CONFIG_USER_ONLY)
1873 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
1874 uint8_t *buf, int len, int is_write)
1875 {
1876 int l, flags;
1877 target_ulong page;
1878 void * p;
1879
1880 while (len > 0) {
1881 page = addr & TARGET_PAGE_MASK;
1882 l = (page + TARGET_PAGE_SIZE) - addr;
1883 if (l > len)
1884 l = len;
1885 flags = page_get_flags(page);
1886 if (!(flags & PAGE_VALID))
1887 return -1;
1888 if (is_write) {
1889 if (!(flags & PAGE_WRITE))
1890 return -1;
1891 /* XXX: this code should not depend on lock_user */
1892 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
1893 return -1;
1894 memcpy(p, buf, l);
1895 unlock_user(p, addr, l);
1896 } else {
1897 if (!(flags & PAGE_READ))
1898 return -1;
1899 /* XXX: this code should not depend on lock_user */
1900 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
1901 return -1;
1902 memcpy(buf, p, l);
1903 unlock_user(p, addr, 0);
1904 }
1905 len -= l;
1906 buf += l;
1907 addr += l;
1908 }
1909 return 0;
1910 }
1911
1912 #else
1913
1914 static void invalidate_and_set_dirty(hwaddr addr,
1915 hwaddr length)
1916 {
1917 if (!cpu_physical_memory_is_dirty(addr)) {
1918 /* invalidate code */
1919 tb_invalidate_phys_page_range(addr, addr + length, 0);
1920 /* set dirty bit */
1921 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_VGA);
1922 cpu_physical_memory_set_dirty_flag(addr, DIRTY_MEMORY_MIGRATION);
1923 }
1924 xen_modified_memory(addr, length);
1925 }
1926
1927 static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
1928 {
1929 if (memory_region_is_ram(mr)) {
1930 return !(is_write && mr->readonly);
1931 }
1932 if (memory_region_is_romd(mr)) {
1933 return !is_write;
1934 }
1935
1936 return false;
1937 }
1938
1939 static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr)
1940 {
1941 unsigned access_size_max = mr->ops->valid.max_access_size;
1942
1943 /* Regions are assumed to support 1-4 byte accesses unless
1944 otherwise specified. */
1945 if (access_size_max == 0) {
1946 access_size_max = 4;
1947 }
1948
1949 /* Bound the maximum access by the alignment of the address. */
1950 if (!mr->ops->impl.unaligned) {
1951 unsigned align_size_max = addr & -addr;
1952 if (align_size_max != 0 && align_size_max < access_size_max) {
1953 access_size_max = align_size_max;
1954 }
1955 }
1956
1957 /* Don't attempt accesses larger than the maximum. */
1958 if (l > access_size_max) {
1959 l = access_size_max;
1960 }
1961 if (l & (l - 1)) {
1962 l = 1 << (qemu_fls(l) - 1);
1963 }
1964
1965 return l;
1966 }
1967
1968 bool address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
1969 int len, bool is_write)
1970 {
1971 hwaddr l;
1972 uint8_t *ptr;
1973 uint64_t val;
1974 hwaddr addr1;
1975 MemoryRegion *mr;
1976 bool error = false;
1977
1978 while (len > 0) {
1979 l = len;
1980 mr = address_space_translate(as, addr, &addr1, &l, is_write);
1981
1982 if (is_write) {
1983 if (!memory_access_is_direct(mr, is_write)) {
1984 l = memory_access_size(mr, l, addr1);
1985 /* XXX: could force current_cpu to NULL to avoid
1986 potential bugs */
1987 switch (l) {
1988 case 8:
1989 /* 64 bit write access */
1990 val = ldq_p(buf);
1991 error |= io_mem_write(mr, addr1, val, 8);
1992 break;
1993 case 4:
1994 /* 32 bit write access */
1995 val = ldl_p(buf);
1996 error |= io_mem_write(mr, addr1, val, 4);
1997 break;
1998 case 2:
1999 /* 16 bit write access */
2000 val = lduw_p(buf);
2001 error |= io_mem_write(mr, addr1, val, 2);
2002 break;
2003 case 1:
2004 /* 8 bit write access */
2005 val = ldub_p(buf);
2006 error |= io_mem_write(mr, addr1, val, 1);
2007 break;
2008 default:
2009 abort();
2010 }
2011 } else {
2012 addr1 += memory_region_get_ram_addr(mr);
2013 /* RAM case */
2014 ptr = qemu_get_ram_ptr(addr1);
2015 memcpy(ptr, buf, l);
2016 invalidate_and_set_dirty(addr1, l);
2017 }
2018 } else {
2019 if (!memory_access_is_direct(mr, is_write)) {
2020 /* I/O case */
2021 l = memory_access_size(mr, l, addr1);
2022 switch (l) {
2023 case 8:
2024 /* 64 bit read access */
2025 error |= io_mem_read(mr, addr1, &val, 8);
2026 stq_p(buf, val);
2027 break;
2028 case 4:
2029 /* 32 bit read access */
2030 error |= io_mem_read(mr, addr1, &val, 4);
2031 stl_p(buf, val);
2032 break;
2033 case 2:
2034 /* 16 bit read access */
2035 error |= io_mem_read(mr, addr1, &val, 2);
2036 stw_p(buf, val);
2037 break;
2038 case 1:
2039 /* 8 bit read access */
2040 error |= io_mem_read(mr, addr1, &val, 1);
2041 stb_p(buf, val);
2042 break;
2043 default:
2044 abort();
2045 }
2046 } else {
2047 /* RAM case */
2048 ptr = qemu_get_ram_ptr(mr->ram_addr + addr1);
2049 memcpy(buf, ptr, l);
2050 }
2051 }
2052 len -= l;
2053 buf += l;
2054 addr += l;
2055 }
2056
2057 return error;
2058 }
2059
2060 bool address_space_write(AddressSpace *as, hwaddr addr,
2061 const uint8_t *buf, int len)
2062 {
2063 return address_space_rw(as, addr, (uint8_t *)buf, len, true);
2064 }
2065
2066 bool address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
2067 {
2068 return address_space_rw(as, addr, buf, len, false);
2069 }
2070
2071
2072 void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
2073 int len, int is_write)
2074 {
2075 address_space_rw(&address_space_memory, addr, buf, len, is_write);
2076 }
2077
2078 enum write_rom_type {
2079 WRITE_DATA,
2080 FLUSH_CACHE,
2081 };
2082
2083 static inline void cpu_physical_memory_write_rom_internal(
2084 hwaddr addr, const uint8_t *buf, int len, enum write_rom_type type)
2085 {
2086 hwaddr l;
2087 uint8_t *ptr;
2088 hwaddr addr1;
2089 MemoryRegion *mr;
2090
2091 while (len > 0) {
2092 l = len;
2093 mr = address_space_translate(&address_space_memory,
2094 addr, &addr1, &l, true);
2095
2096 if (!(memory_region_is_ram(mr) ||
2097 memory_region_is_romd(mr))) {
2098 /* do nothing */
2099 } else {
2100 addr1 += memory_region_get_ram_addr(mr);
2101 /* ROM/RAM case */
2102 ptr = qemu_get_ram_ptr(addr1);
2103 switch (type) {
2104 case WRITE_DATA:
2105 memcpy(ptr, buf, l);
2106 invalidate_and_set_dirty(addr1, l);
2107 break;
2108 case FLUSH_CACHE:
2109 flush_icache_range((uintptr_t)ptr, (uintptr_t)ptr + l);
2110 break;
2111 }
2112 }
2113 len -= l;
2114 buf += l;
2115 addr += l;
2116 }
2117 }
2118
2119 /* used for ROM loading : can write in RAM and ROM */
2120 void cpu_physical_memory_write_rom(hwaddr addr,
2121 const uint8_t *buf, int len)
2122 {
2123 cpu_physical_memory_write_rom_internal(addr, buf, len, WRITE_DATA);
2124 }
2125
2126 void cpu_flush_icache_range(hwaddr start, int len)
2127 {
2128 /*
2129 * This function should do the same thing as an icache flush that was
2130 * triggered from within the guest. For TCG we are always cache coherent,
2131 * so there is no need to flush anything. For KVM / Xen we need to flush
2132 * the host's instruction cache at least.
2133 */
2134 if (tcg_enabled()) {
2135 return;
2136 }
2137
2138 cpu_physical_memory_write_rom_internal(start, NULL, len, FLUSH_CACHE);
2139 }
2140
2141 typedef struct {
2142 MemoryRegion *mr;
2143 void *buffer;
2144 hwaddr addr;
2145 hwaddr len;
2146 } BounceBuffer;
2147
2148 static BounceBuffer bounce;
2149
2150 typedef struct MapClient {
2151 void *opaque;
2152 void (*callback)(void *opaque);
2153 QLIST_ENTRY(MapClient) link;
2154 } MapClient;
2155
2156 static QLIST_HEAD(map_client_list, MapClient) map_client_list
2157 = QLIST_HEAD_INITIALIZER(map_client_list);
2158
2159 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
2160 {
2161 MapClient *client = g_malloc(sizeof(*client));
2162
2163 client->opaque = opaque;
2164 client->callback = callback;
2165 QLIST_INSERT_HEAD(&map_client_list, client, link);
2166 return client;
2167 }
2168
2169 static void cpu_unregister_map_client(void *_client)
2170 {
2171 MapClient *client = (MapClient *)_client;
2172
2173 QLIST_REMOVE(client, link);
2174 g_free(client);
2175 }
2176
2177 static void cpu_notify_map_clients(void)
2178 {
2179 MapClient *client;
2180
2181 while (!QLIST_EMPTY(&map_client_list)) {
2182 client = QLIST_FIRST(&map_client_list);
2183 client->callback(client->opaque);
2184 cpu_unregister_map_client(client);
2185 }
2186 }
2187
2188 bool address_space_access_valid(AddressSpace *as, hwaddr addr, int len, bool is_write)
2189 {
2190 MemoryRegion *mr;
2191 hwaddr l, xlat;
2192
2193 while (len > 0) {
2194 l = len;
2195 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2196 if (!memory_access_is_direct(mr, is_write)) {
2197 l = memory_access_size(mr, l, addr);
2198 if (!memory_region_access_valid(mr, xlat, l, is_write)) {
2199 return false;
2200 }
2201 }
2202
2203 len -= l;
2204 addr += l;
2205 }
2206 return true;
2207 }
2208
2209 /* Map a physical memory region into a host virtual address.
2210 * May map a subset of the requested range, given by and returned in *plen.
2211 * May return NULL if resources needed to perform the mapping are exhausted.
2212 * Use only for reads OR writes - not for read-modify-write operations.
2213 * Use cpu_register_map_client() to know when retrying the map operation is
2214 * likely to succeed.
2215 */
2216 void *address_space_map(AddressSpace *as,
2217 hwaddr addr,
2218 hwaddr *plen,
2219 bool is_write)
2220 {
2221 hwaddr len = *plen;
2222 hwaddr done = 0;
2223 hwaddr l, xlat, base;
2224 MemoryRegion *mr, *this_mr;
2225 ram_addr_t raddr;
2226
2227 if (len == 0) {
2228 return NULL;
2229 }
2230
2231 l = len;
2232 mr = address_space_translate(as, addr, &xlat, &l, is_write);
2233 if (!memory_access_is_direct(mr, is_write)) {
2234 if (bounce.buffer) {
2235 return NULL;
2236 }
2237 /* Avoid unbounded allocations */
2238 l = MIN(l, TARGET_PAGE_SIZE);
2239 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, l);
2240 bounce.addr = addr;
2241 bounce.len = l;
2242
2243 memory_region_ref(mr);
2244 bounce.mr = mr;
2245 if (!is_write) {
2246 address_space_read(as, addr, bounce.buffer, l);
2247 }
2248
2249 *plen = l;
2250 return bounce.buffer;
2251 }
2252
2253 base = xlat;
2254 raddr = memory_region_get_ram_addr(mr);
2255
2256 for (;;) {
2257 len -= l;
2258 addr += l;
2259 done += l;
2260 if (len == 0) {
2261 break;
2262 }
2263
2264 l = len;
2265 this_mr = address_space_translate(as, addr, &xlat, &l, is_write);
2266 if (this_mr != mr || xlat != base + done) {
2267 break;
2268 }
2269 }
2270
2271 memory_region_ref(mr);
2272 *plen = done;
2273 return qemu_ram_ptr_length(raddr + base, plen);
2274 }
2275
2276 /* Unmaps a memory region previously mapped by address_space_map().
2277 * Will also mark the memory as dirty if is_write == 1. access_len gives
2278 * the amount of memory that was actually read or written by the caller.
2279 */
2280 void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
2281 int is_write, hwaddr access_len)
2282 {
2283 if (buffer != bounce.buffer) {
2284 MemoryRegion *mr;
2285 ram_addr_t addr1;
2286
2287 mr = qemu_ram_addr_from_host(buffer, &addr1);
2288 assert(mr != NULL);
2289 if (is_write) {
2290 while (access_len) {
2291 unsigned l;
2292 l = TARGET_PAGE_SIZE;
2293 if (l > access_len)
2294 l = access_len;
2295 invalidate_and_set_dirty(addr1, l);
2296 addr1 += l;
2297 access_len -= l;
2298 }
2299 }
2300 if (xen_enabled()) {
2301 xen_invalidate_map_cache_entry(buffer);
2302 }
2303 memory_region_unref(mr);
2304 return;
2305 }
2306 if (is_write) {
2307 address_space_write(as, bounce.addr, bounce.buffer, access_len);
2308 }
2309 qemu_vfree(bounce.buffer);
2310 bounce.buffer = NULL;
2311 memory_region_unref(bounce.mr);
2312 cpu_notify_map_clients();
2313 }
2314
2315 void *cpu_physical_memory_map(hwaddr addr,
2316 hwaddr *plen,
2317 int is_write)
2318 {
2319 return address_space_map(&address_space_memory, addr, plen, is_write);
2320 }
2321
2322 void cpu_physical_memory_unmap(void *buffer, hwaddr len,
2323 int is_write, hwaddr access_len)
2324 {
2325 return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
2326 }
2327
2328 /* warning: addr must be aligned */
2329 static inline uint32_t ldl_phys_internal(hwaddr addr,
2330 enum device_endian endian)
2331 {
2332 uint8_t *ptr;
2333 uint64_t val;
2334 MemoryRegion *mr;
2335 hwaddr l = 4;
2336 hwaddr addr1;
2337
2338 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2339 false);
2340 if (l < 4 || !memory_access_is_direct(mr, false)) {
2341 /* I/O case */
2342 io_mem_read(mr, addr1, &val, 4);
2343 #if defined(TARGET_WORDS_BIGENDIAN)
2344 if (endian == DEVICE_LITTLE_ENDIAN) {
2345 val = bswap32(val);
2346 }
2347 #else
2348 if (endian == DEVICE_BIG_ENDIAN) {
2349 val = bswap32(val);
2350 }
2351 #endif
2352 } else {
2353 /* RAM case */
2354 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2355 & TARGET_PAGE_MASK)
2356 + addr1);
2357 switch (endian) {
2358 case DEVICE_LITTLE_ENDIAN:
2359 val = ldl_le_p(ptr);
2360 break;
2361 case DEVICE_BIG_ENDIAN:
2362 val = ldl_be_p(ptr);
2363 break;
2364 default:
2365 val = ldl_p(ptr);
2366 break;
2367 }
2368 }
2369 return val;
2370 }
2371
2372 uint32_t ldl_phys(hwaddr addr)
2373 {
2374 return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2375 }
2376
2377 uint32_t ldl_le_phys(hwaddr addr)
2378 {
2379 return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2380 }
2381
2382 uint32_t ldl_be_phys(hwaddr addr)
2383 {
2384 return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
2385 }
2386
2387 /* warning: addr must be aligned */
2388 static inline uint64_t ldq_phys_internal(hwaddr addr,
2389 enum device_endian endian)
2390 {
2391 uint8_t *ptr;
2392 uint64_t val;
2393 MemoryRegion *mr;
2394 hwaddr l = 8;
2395 hwaddr addr1;
2396
2397 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2398 false);
2399 if (l < 8 || !memory_access_is_direct(mr, false)) {
2400 /* I/O case */
2401 io_mem_read(mr, addr1, &val, 8);
2402 #if defined(TARGET_WORDS_BIGENDIAN)
2403 if (endian == DEVICE_LITTLE_ENDIAN) {
2404 val = bswap64(val);
2405 }
2406 #else
2407 if (endian == DEVICE_BIG_ENDIAN) {
2408 val = bswap64(val);
2409 }
2410 #endif
2411 } else {
2412 /* RAM case */
2413 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2414 & TARGET_PAGE_MASK)
2415 + addr1);
2416 switch (endian) {
2417 case DEVICE_LITTLE_ENDIAN:
2418 val = ldq_le_p(ptr);
2419 break;
2420 case DEVICE_BIG_ENDIAN:
2421 val = ldq_be_p(ptr);
2422 break;
2423 default:
2424 val = ldq_p(ptr);
2425 break;
2426 }
2427 }
2428 return val;
2429 }
2430
2431 uint64_t ldq_phys(hwaddr addr)
2432 {
2433 return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2434 }
2435
2436 uint64_t ldq_le_phys(hwaddr addr)
2437 {
2438 return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2439 }
2440
2441 uint64_t ldq_be_phys(hwaddr addr)
2442 {
2443 return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
2444 }
2445
2446 /* XXX: optimize */
2447 uint32_t ldub_phys(hwaddr addr)
2448 {
2449 uint8_t val;
2450 cpu_physical_memory_read(addr, &val, 1);
2451 return val;
2452 }
2453
2454 /* warning: addr must be aligned */
2455 static inline uint32_t lduw_phys_internal(hwaddr addr,
2456 enum device_endian endian)
2457 {
2458 uint8_t *ptr;
2459 uint64_t val;
2460 MemoryRegion *mr;
2461 hwaddr l = 2;
2462 hwaddr addr1;
2463
2464 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2465 false);
2466 if (l < 2 || !memory_access_is_direct(mr, false)) {
2467 /* I/O case */
2468 io_mem_read(mr, addr1, &val, 2);
2469 #if defined(TARGET_WORDS_BIGENDIAN)
2470 if (endian == DEVICE_LITTLE_ENDIAN) {
2471 val = bswap16(val);
2472 }
2473 #else
2474 if (endian == DEVICE_BIG_ENDIAN) {
2475 val = bswap16(val);
2476 }
2477 #endif
2478 } else {
2479 /* RAM case */
2480 ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(mr)
2481 & TARGET_PAGE_MASK)
2482 + addr1);
2483 switch (endian) {
2484 case DEVICE_LITTLE_ENDIAN:
2485 val = lduw_le_p(ptr);
2486 break;
2487 case DEVICE_BIG_ENDIAN:
2488 val = lduw_be_p(ptr);
2489 break;
2490 default:
2491 val = lduw_p(ptr);
2492 break;
2493 }
2494 }
2495 return val;
2496 }
2497
2498 uint32_t lduw_phys(hwaddr addr)
2499 {
2500 return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
2501 }
2502
2503 uint32_t lduw_le_phys(hwaddr addr)
2504 {
2505 return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
2506 }
2507
2508 uint32_t lduw_be_phys(hwaddr addr)
2509 {
2510 return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
2511 }
2512
2513 /* warning: addr must be aligned. The ram page is not masked as dirty
2514 and the code inside is not invalidated. It is useful if the dirty
2515 bits are used to track modified PTEs */
2516 void stl_phys_notdirty(hwaddr addr, uint32_t val)
2517 {
2518 uint8_t *ptr;
2519 MemoryRegion *mr;
2520 hwaddr l = 4;
2521 hwaddr addr1;
2522
2523 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2524 true);
2525 if (l < 4 || !memory_access_is_direct(mr, true)) {
2526 io_mem_write(mr, addr1, val, 4);
2527 } else {
2528 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2529 ptr = qemu_get_ram_ptr(addr1);
2530 stl_p(ptr, val);
2531
2532 if (unlikely(in_migration)) {
2533 if (!cpu_physical_memory_is_dirty(addr1)) {
2534 /* invalidate code */
2535 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
2536 /* set dirty bit */
2537 cpu_physical_memory_set_dirty_flag(addr1,
2538 DIRTY_MEMORY_MIGRATION);
2539 cpu_physical_memory_set_dirty_flag(addr1, DIRTY_MEMORY_VGA);
2540 }
2541 }
2542 }
2543 }
2544
2545 /* warning: addr must be aligned */
2546 static inline void stl_phys_internal(hwaddr addr, uint32_t val,
2547 enum device_endian endian)
2548 {
2549 uint8_t *ptr;
2550 MemoryRegion *mr;
2551 hwaddr l = 4;
2552 hwaddr addr1;
2553
2554 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2555 true);
2556 if (l < 4 || !memory_access_is_direct(mr, true)) {
2557 #if defined(TARGET_WORDS_BIGENDIAN)
2558 if (endian == DEVICE_LITTLE_ENDIAN) {
2559 val = bswap32(val);
2560 }
2561 #else
2562 if (endian == DEVICE_BIG_ENDIAN) {
2563 val = bswap32(val);
2564 }
2565 #endif
2566 io_mem_write(mr, addr1, val, 4);
2567 } else {
2568 /* RAM case */
2569 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2570 ptr = qemu_get_ram_ptr(addr1);
2571 switch (endian) {
2572 case DEVICE_LITTLE_ENDIAN:
2573 stl_le_p(ptr, val);
2574 break;
2575 case DEVICE_BIG_ENDIAN:
2576 stl_be_p(ptr, val);
2577 break;
2578 default:
2579 stl_p(ptr, val);
2580 break;
2581 }
2582 invalidate_and_set_dirty(addr1, 4);
2583 }
2584 }
2585
2586 void stl_phys(hwaddr addr, uint32_t val)
2587 {
2588 stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2589 }
2590
2591 void stl_le_phys(hwaddr addr, uint32_t val)
2592 {
2593 stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2594 }
2595
2596 void stl_be_phys(hwaddr addr, uint32_t val)
2597 {
2598 stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2599 }
2600
2601 /* XXX: optimize */
2602 void stb_phys(hwaddr addr, uint32_t val)
2603 {
2604 uint8_t v = val;
2605 cpu_physical_memory_write(addr, &v, 1);
2606 }
2607
2608 /* warning: addr must be aligned */
2609 static inline void stw_phys_internal(hwaddr addr, uint32_t val,
2610 enum device_endian endian)
2611 {
2612 uint8_t *ptr;
2613 MemoryRegion *mr;
2614 hwaddr l = 2;
2615 hwaddr addr1;
2616
2617 mr = address_space_translate(&address_space_memory, addr, &addr1, &l,
2618 true);
2619 if (l < 2 || !memory_access_is_direct(mr, true)) {
2620 #if defined(TARGET_WORDS_BIGENDIAN)
2621 if (endian == DEVICE_LITTLE_ENDIAN) {
2622 val = bswap16(val);
2623 }
2624 #else
2625 if (endian == DEVICE_BIG_ENDIAN) {
2626 val = bswap16(val);
2627 }
2628 #endif
2629 io_mem_write(mr, addr1, val, 2);
2630 } else {
2631 /* RAM case */
2632 addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK;
2633 ptr = qemu_get_ram_ptr(addr1);
2634 switch (endian) {
2635 case DEVICE_LITTLE_ENDIAN:
2636 stw_le_p(ptr, val);
2637 break;
2638 case DEVICE_BIG_ENDIAN:
2639 stw_be_p(ptr, val);
2640 break;
2641 default:
2642 stw_p(ptr, val);
2643 break;
2644 }
2645 invalidate_and_set_dirty(addr1, 2);
2646 }
2647 }
2648
2649 void stw_phys(hwaddr addr, uint32_t val)
2650 {
2651 stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
2652 }
2653
2654 void stw_le_phys(hwaddr addr, uint32_t val)
2655 {
2656 stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
2657 }
2658
2659 void stw_be_phys(hwaddr addr, uint32_t val)
2660 {
2661 stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
2662 }
2663
2664 /* XXX: optimize */
2665 void stq_phys(hwaddr addr, uint64_t val)
2666 {
2667 val = tswap64(val);
2668 cpu_physical_memory_write(addr, &val, 8);
2669 }
2670
2671 void stq_le_phys(hwaddr addr, uint64_t val)
2672 {
2673 val = cpu_to_le64(val);
2674 cpu_physical_memory_write(addr, &val, 8);
2675 }
2676
2677 void stq_be_phys(hwaddr addr, uint64_t val)
2678 {
2679 val = cpu_to_be64(val);
2680 cpu_physical_memory_write(addr, &val, 8);
2681 }
2682
2683 /* virtual memory access for debug (includes writing to ROM) */
2684 int cpu_memory_rw_debug(CPUState *cpu, target_ulong addr,
2685 uint8_t *buf, int len, int is_write)
2686 {
2687 int l;
2688 hwaddr phys_addr;
2689 target_ulong page;
2690
2691 while (len > 0) {
2692 page = addr & TARGET_PAGE_MASK;
2693 phys_addr = cpu_get_phys_page_debug(cpu, page);
2694 /* if no physical page mapped, return an error */
2695 if (phys_addr == -1)
2696 return -1;
2697 l = (page + TARGET_PAGE_SIZE) - addr;
2698 if (l > len)
2699 l = len;
2700 phys_addr += (addr & ~TARGET_PAGE_MASK);
2701 if (is_write)
2702 cpu_physical_memory_write_rom(phys_addr, buf, l);
2703 else
2704 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
2705 len -= l;
2706 buf += l;
2707 addr += l;
2708 }
2709 return 0;
2710 }
2711 #endif
2712
2713 #if !defined(CONFIG_USER_ONLY)
2714
2715 /*
2716 * A helper function for the _utterly broken_ virtio device model to find out if
2717 * it's running on a big endian machine. Don't do this at home kids!
2718 */
2719 bool virtio_is_big_endian(void);
2720 bool virtio_is_big_endian(void)
2721 {
2722 #if defined(TARGET_WORDS_BIGENDIAN)
2723 return true;
2724 #else
2725 return false;
2726 #endif
2727 }
2728
2729 #endif
2730
2731 #ifndef CONFIG_USER_ONLY
2732 bool cpu_physical_memory_is_io(hwaddr phys_addr)
2733 {
2734 MemoryRegion*mr;
2735 hwaddr l = 1;
2736
2737 mr = address_space_translate(&address_space_memory,
2738 phys_addr, &phys_addr, &l, false);
2739
2740 return !(memory_region_is_ram(mr) ||
2741 memory_region_is_romd(mr));
2742 }
2743
2744 void qemu_ram_foreach_block(RAMBlockIterFunc func, void *opaque)
2745 {
2746 RAMBlock *block;
2747
2748 QTAILQ_FOREACH(block, &ram_list.blocks, next) {
2749 func(block->host, block->offset, block->length, opaque);
2750 }
2751 }
2752 #endif
QEmu