virtfs: fix build due from rename
[qemu.git] / exec.c
blobc3dc68ae092509e2abdba4d2ca97b2ecd9a4aa19
1 /*
2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
27 #include "qemu-common.h"
28 #include "cpu.h"
29 #include "exec-all.h"
30 #include "tcg.h"
31 #include "hw/hw.h"
32 #include "hw/qdev.h"
33 #include "osdep.h"
34 #include "kvm.h"
35 #include "qemu-timer.h"
36 #if defined(CONFIG_USER_ONLY)
37 #include <qemu.h>
38 #include <signal.h>
39 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
40 #include <sys/param.h>
41 #if __FreeBSD_version >= 700104
42 #define HAVE_KINFO_GETVMMAP
43 #define sigqueue sigqueue_freebsd /* avoid redefinition */
44 #include <sys/time.h>
45 #include <sys/proc.h>
46 #include <machine/profile.h>
47 #define _KERNEL
48 #include <sys/user.h>
49 #undef _KERNEL
50 #undef sigqueue
51 #include <libutil.h>
52 #endif
53 #endif
54 #endif
56 //#define DEBUG_TB_INVALIDATE
57 //#define DEBUG_FLUSH
58 //#define DEBUG_TLB
59 //#define DEBUG_UNASSIGNED
61 /* make various TB consistency checks */
62 //#define DEBUG_TB_CHECK
63 //#define DEBUG_TLB_CHECK
65 //#define DEBUG_IOPORT
66 //#define DEBUG_SUBPAGE
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
70 #undef DEBUG_TB_CHECK
71 #endif
73 #define SMC_BITMAP_USE_THRESHOLD 10
75 static TranslationBlock *tbs;
76 static int code_gen_max_blocks;
77 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
78 static int nb_tbs;
79 /* any access to the tbs or the page table must use this lock */
80 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
82 #if defined(__arm__) || defined(__sparc_v9__)
83 /* The prologue must be reachable with a direct jump. ARM and Sparc64
84 have limited branch ranges (possibly also PPC) so place it in a
85 section close to code segment. */
86 #define code_gen_section \
87 __attribute__((__section__(".gen_code"))) \
88 __attribute__((aligned (32)))
89 #elif defined(_WIN32)
90 /* Maximum alignment for Win32 is 16. */
91 #define code_gen_section \
92 __attribute__((aligned (16)))
93 #else
94 #define code_gen_section \
95 __attribute__((aligned (32)))
96 #endif
98 uint8_t code_gen_prologue[1024] code_gen_section;
99 static uint8_t *code_gen_buffer;
100 static unsigned long code_gen_buffer_size;
101 /* threshold to flush the translated code buffer */
102 static unsigned long code_gen_buffer_max_size;
103 static uint8_t *code_gen_ptr;
105 #if !defined(CONFIG_USER_ONLY)
106 int phys_ram_fd;
107 static int in_migration;
109 RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list) };
110 #endif
112 CPUState *first_cpu;
113 /* current CPU in the current thread. It is only valid inside
114 cpu_exec() */
115 CPUState *cpu_single_env;
116 /* 0 = Do not count executed instructions.
117 1 = Precise instruction counting.
118 2 = Adaptive rate instruction counting. */
119 int use_icount = 0;
120 /* Current instruction counter. While executing translated code this may
121 include some instructions that have not yet been executed. */
122 int64_t qemu_icount;
124 typedef struct PageDesc {
125 /* list of TBs intersecting this ram page */
126 TranslationBlock *first_tb;
127 /* in order to optimize self modifying code, we count the number
128 of lookups we do to a given page to use a bitmap */
129 unsigned int code_write_count;
130 uint8_t *code_bitmap;
131 #if defined(CONFIG_USER_ONLY)
132 unsigned long flags;
133 #endif
134 } PageDesc;
136 /* In system mode we want L1_MAP to be based on ram offsets,
137 while in user mode we want it to be based on virtual addresses. */
138 #if !defined(CONFIG_USER_ONLY)
139 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
140 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
141 #else
142 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
143 #endif
144 #else
145 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
146 #endif
148 /* Size of the L2 (and L3, etc) page tables. */
149 #define L2_BITS 10
150 #define L2_SIZE (1 << L2_BITS)
152 /* The bits remaining after N lower levels of page tables. */
153 #define P_L1_BITS_REM \
154 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
155 #define V_L1_BITS_REM \
156 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
158 /* Size of the L1 page table. Avoid silly small sizes. */
159 #if P_L1_BITS_REM < 4
160 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
161 #else
162 #define P_L1_BITS P_L1_BITS_REM
163 #endif
165 #if V_L1_BITS_REM < 4
166 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
167 #else
168 #define V_L1_BITS V_L1_BITS_REM
169 #endif
171 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
172 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
174 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
175 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
177 unsigned long qemu_real_host_page_size;
178 unsigned long qemu_host_page_bits;
179 unsigned long qemu_host_page_size;
180 unsigned long qemu_host_page_mask;
182 /* This is a multi-level map on the virtual address space.
183 The bottom level has pointers to PageDesc. */
184 static void *l1_map[V_L1_SIZE];
186 #if !defined(CONFIG_USER_ONLY)
187 typedef struct PhysPageDesc {
188 /* offset in host memory of the page + io_index in the low bits */
189 ram_addr_t phys_offset;
190 ram_addr_t region_offset;
191 } PhysPageDesc;
193 /* This is a multi-level map on the physical address space.
194 The bottom level has pointers to PhysPageDesc. */
195 static void *l1_phys_map[P_L1_SIZE];
197 static void io_mem_init(void);
199 /* io memory support */
200 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
201 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
202 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
203 static char io_mem_used[IO_MEM_NB_ENTRIES];
204 static int io_mem_watch;
205 #endif
207 /* log support */
208 #ifdef WIN32
209 static const char *logfilename = "qemu.log";
210 #else
211 static const char *logfilename = "/tmp/qemu.log";
212 #endif
213 FILE *logfile;
214 int loglevel;
215 static int log_append = 0;
217 /* statistics */
218 #if !defined(CONFIG_USER_ONLY)
219 static int tlb_flush_count;
220 #endif
221 static int tb_flush_count;
222 static int tb_phys_invalidate_count;
224 #ifdef _WIN32
225 static void map_exec(void *addr, long size)
227 DWORD old_protect;
228 VirtualProtect(addr, size,
229 PAGE_EXECUTE_READWRITE, &old_protect);
232 #else
233 static void map_exec(void *addr, long size)
235 unsigned long start, end, page_size;
237 page_size = getpagesize();
238 start = (unsigned long)addr;
239 start &= ~(page_size - 1);
241 end = (unsigned long)addr + size;
242 end += page_size - 1;
243 end &= ~(page_size - 1);
245 mprotect((void *)start, end - start,
246 PROT_READ | PROT_WRITE | PROT_EXEC);
248 #endif
250 static void page_init(void)
252 /* NOTE: we can always suppose that qemu_host_page_size >=
253 TARGET_PAGE_SIZE */
254 #ifdef _WIN32
256 SYSTEM_INFO system_info;
258 GetSystemInfo(&system_info);
259 qemu_real_host_page_size = system_info.dwPageSize;
261 #else
262 qemu_real_host_page_size = getpagesize();
263 #endif
264 if (qemu_host_page_size == 0)
265 qemu_host_page_size = qemu_real_host_page_size;
266 if (qemu_host_page_size < TARGET_PAGE_SIZE)
267 qemu_host_page_size = TARGET_PAGE_SIZE;
268 qemu_host_page_bits = 0;
269 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
270 qemu_host_page_bits++;
271 qemu_host_page_mask = ~(qemu_host_page_size - 1);
273 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
275 #ifdef HAVE_KINFO_GETVMMAP
276 struct kinfo_vmentry *freep;
277 int i, cnt;
279 freep = kinfo_getvmmap(getpid(), &cnt);
280 if (freep) {
281 mmap_lock();
282 for (i = 0; i < cnt; i++) {
283 unsigned long startaddr, endaddr;
285 startaddr = freep[i].kve_start;
286 endaddr = freep[i].kve_end;
287 if (h2g_valid(startaddr)) {
288 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
290 if (h2g_valid(endaddr)) {
291 endaddr = h2g(endaddr);
292 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
293 } else {
294 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
295 endaddr = ~0ul;
296 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
297 #endif
301 free(freep);
302 mmap_unlock();
304 #else
305 FILE *f;
307 last_brk = (unsigned long)sbrk(0);
309 f = fopen("/compat/linux/proc/self/maps", "r");
310 if (f) {
311 mmap_lock();
313 do {
314 unsigned long startaddr, endaddr;
315 int n;
317 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
319 if (n == 2 && h2g_valid(startaddr)) {
320 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
322 if (h2g_valid(endaddr)) {
323 endaddr = h2g(endaddr);
324 } else {
325 endaddr = ~0ul;
327 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
329 } while (!feof(f));
331 fclose(f);
332 mmap_unlock();
334 #endif
336 #endif
339 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
341 PageDesc *pd;
342 void **lp;
343 int i;
345 #if defined(CONFIG_USER_ONLY)
346 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
347 # define ALLOC(P, SIZE) \
348 do { \
349 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
350 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
351 } while (0)
352 #else
353 # define ALLOC(P, SIZE) \
354 do { P = qemu_mallocz(SIZE); } while (0)
355 #endif
357 /* Level 1. Always allocated. */
358 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
360 /* Level 2..N-1. */
361 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
362 void **p = *lp;
364 if (p == NULL) {
365 if (!alloc) {
366 return NULL;
368 ALLOC(p, sizeof(void *) * L2_SIZE);
369 *lp = p;
372 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
375 pd = *lp;
376 if (pd == NULL) {
377 if (!alloc) {
378 return NULL;
380 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
381 *lp = pd;
384 #undef ALLOC
386 return pd + (index & (L2_SIZE - 1));
389 static inline PageDesc *page_find(tb_page_addr_t index)
391 return page_find_alloc(index, 0);
394 #if !defined(CONFIG_USER_ONLY)
395 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
397 PhysPageDesc *pd;
398 void **lp;
399 int i;
401 /* Level 1. Always allocated. */
402 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
404 /* Level 2..N-1. */
405 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
406 void **p = *lp;
407 if (p == NULL) {
408 if (!alloc) {
409 return NULL;
411 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
413 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
416 pd = *lp;
417 if (pd == NULL) {
418 int i;
420 if (!alloc) {
421 return NULL;
424 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
426 for (i = 0; i < L2_SIZE; i++) {
427 pd[i].phys_offset = IO_MEM_UNASSIGNED;
428 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
432 return pd + (index & (L2_SIZE - 1));
435 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
437 return phys_page_find_alloc(index, 0);
440 static void tlb_protect_code(ram_addr_t ram_addr);
441 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
442 target_ulong vaddr);
443 #define mmap_lock() do { } while(0)
444 #define mmap_unlock() do { } while(0)
445 #endif
447 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
449 #if defined(CONFIG_USER_ONLY)
450 /* Currently it is not recommended to allocate big chunks of data in
451 user mode. It will change when a dedicated libc will be used */
452 #define USE_STATIC_CODE_GEN_BUFFER
453 #endif
455 #ifdef USE_STATIC_CODE_GEN_BUFFER
456 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
457 __attribute__((aligned (CODE_GEN_ALIGN)));
458 #endif
460 static void code_gen_alloc(unsigned long tb_size)
462 #ifdef USE_STATIC_CODE_GEN_BUFFER
463 code_gen_buffer = static_code_gen_buffer;
464 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
465 map_exec(code_gen_buffer, code_gen_buffer_size);
466 #else
467 code_gen_buffer_size = tb_size;
468 if (code_gen_buffer_size == 0) {
469 #if defined(CONFIG_USER_ONLY)
470 /* in user mode, phys_ram_size is not meaningful */
471 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
472 #else
473 /* XXX: needs adjustments */
474 code_gen_buffer_size = (unsigned long)(ram_size / 4);
475 #endif
477 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
478 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
479 /* The code gen buffer location may have constraints depending on
480 the host cpu and OS */
481 #if defined(__linux__)
483 int flags;
484 void *start = NULL;
486 flags = MAP_PRIVATE | MAP_ANONYMOUS;
487 #if defined(__x86_64__)
488 flags |= MAP_32BIT;
489 /* Cannot map more than that */
490 if (code_gen_buffer_size > (800 * 1024 * 1024))
491 code_gen_buffer_size = (800 * 1024 * 1024);
492 #elif defined(__sparc_v9__)
493 // Map the buffer below 2G, so we can use direct calls and branches
494 flags |= MAP_FIXED;
495 start = (void *) 0x60000000UL;
496 if (code_gen_buffer_size > (512 * 1024 * 1024))
497 code_gen_buffer_size = (512 * 1024 * 1024);
498 #elif defined(__arm__)
499 /* Map the buffer below 32M, so we can use direct calls and branches */
500 flags |= MAP_FIXED;
501 start = (void *) 0x01000000UL;
502 if (code_gen_buffer_size > 16 * 1024 * 1024)
503 code_gen_buffer_size = 16 * 1024 * 1024;
504 #elif defined(__s390x__)
505 /* Map the buffer so that we can use direct calls and branches. */
506 /* We have a +- 4GB range on the branches; leave some slop. */
507 if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
508 code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
510 start = (void *)0x90000000UL;
511 #endif
512 code_gen_buffer = mmap(start, code_gen_buffer_size,
513 PROT_WRITE | PROT_READ | PROT_EXEC,
514 flags, -1, 0);
515 if (code_gen_buffer == MAP_FAILED) {
516 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
517 exit(1);
520 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
521 || defined(__DragonFly__) || defined(__OpenBSD__)
523 int flags;
524 void *addr = NULL;
525 flags = MAP_PRIVATE | MAP_ANONYMOUS;
526 #if defined(__x86_64__)
527 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
528 * 0x40000000 is free */
529 flags |= MAP_FIXED;
530 addr = (void *)0x40000000;
531 /* Cannot map more than that */
532 if (code_gen_buffer_size > (800 * 1024 * 1024))
533 code_gen_buffer_size = (800 * 1024 * 1024);
534 #elif defined(__sparc_v9__)
535 // Map the buffer below 2G, so we can use direct calls and branches
536 flags |= MAP_FIXED;
537 addr = (void *) 0x60000000UL;
538 if (code_gen_buffer_size > (512 * 1024 * 1024)) {
539 code_gen_buffer_size = (512 * 1024 * 1024);
541 #endif
542 code_gen_buffer = mmap(addr, code_gen_buffer_size,
543 PROT_WRITE | PROT_READ | PROT_EXEC,
544 flags, -1, 0);
545 if (code_gen_buffer == MAP_FAILED) {
546 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
547 exit(1);
550 #else
551 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
552 map_exec(code_gen_buffer, code_gen_buffer_size);
553 #endif
554 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
555 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
556 code_gen_buffer_max_size = code_gen_buffer_size -
557 (TCG_MAX_OP_SIZE * OPC_MAX_SIZE);
558 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
559 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
562 /* Must be called before using the QEMU cpus. 'tb_size' is the size
563 (in bytes) allocated to the translation buffer. Zero means default
564 size. */
565 void cpu_exec_init_all(unsigned long tb_size)
567 cpu_gen_init();
568 code_gen_alloc(tb_size);
569 code_gen_ptr = code_gen_buffer;
570 page_init();
571 #if !defined(CONFIG_USER_ONLY)
572 io_mem_init();
573 #endif
574 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
575 /* There's no guest base to take into account, so go ahead and
576 initialize the prologue now. */
577 tcg_prologue_init(&tcg_ctx);
578 #endif
581 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
583 static int cpu_common_post_load(void *opaque, int version_id)
585 CPUState *env = opaque;
587 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
588 version_id is increased. */
589 env->interrupt_request &= ~0x01;
590 tlb_flush(env, 1);
592 return 0;
595 static const VMStateDescription vmstate_cpu_common = {
596 .name = "cpu_common",
597 .version_id = 1,
598 .minimum_version_id = 1,
599 .minimum_version_id_old = 1,
600 .post_load = cpu_common_post_load,
601 .fields = (VMStateField []) {
602 VMSTATE_UINT32(halted, CPUState),
603 VMSTATE_UINT32(interrupt_request, CPUState),
604 VMSTATE_END_OF_LIST()
607 #endif
609 CPUState *qemu_get_cpu(int cpu)
611 CPUState *env = first_cpu;
613 while (env) {
614 if (env->cpu_index == cpu)
615 break;
616 env = env->next_cpu;
619 return env;
622 void cpu_exec_init(CPUState *env)
624 CPUState **penv;
625 int cpu_index;
627 #if defined(CONFIG_USER_ONLY)
628 cpu_list_lock();
629 #endif
630 env->next_cpu = NULL;
631 penv = &first_cpu;
632 cpu_index = 0;
633 while (*penv != NULL) {
634 penv = &(*penv)->next_cpu;
635 cpu_index++;
637 env->cpu_index = cpu_index;
638 env->numa_node = 0;
639 QTAILQ_INIT(&env->breakpoints);
640 QTAILQ_INIT(&env->watchpoints);
641 #ifndef CONFIG_USER_ONLY
642 env->thread_id = qemu_get_thread_id();
643 #endif
644 *penv = env;
645 #if defined(CONFIG_USER_ONLY)
646 cpu_list_unlock();
647 #endif
648 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
649 vmstate_register(NULL, cpu_index, &vmstate_cpu_common, env);
650 register_savevm(NULL, "cpu", cpu_index, CPU_SAVE_VERSION,
651 cpu_save, cpu_load, env);
652 #endif
655 /* Allocate a new translation block. Flush the translation buffer if
656 too many translation blocks or too much generated code. */
657 static TranslationBlock *tb_alloc(target_ulong pc)
659 TranslationBlock *tb;
661 if (nb_tbs >= code_gen_max_blocks ||
662 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
663 return NULL;
664 tb = &tbs[nb_tbs++];
665 tb->pc = pc;
666 tb->cflags = 0;
667 return tb;
670 void tb_free(TranslationBlock *tb)
672 /* In practice this is mostly used for single use temporary TB
673 Ignore the hard cases and just back up if this TB happens to
674 be the last one generated. */
675 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
676 code_gen_ptr = tb->tc_ptr;
677 nb_tbs--;
681 static inline void invalidate_page_bitmap(PageDesc *p)
683 if (p->code_bitmap) {
684 qemu_free(p->code_bitmap);
685 p->code_bitmap = NULL;
687 p->code_write_count = 0;
690 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
692 static void page_flush_tb_1 (int level, void **lp)
694 int i;
696 if (*lp == NULL) {
697 return;
699 if (level == 0) {
700 PageDesc *pd = *lp;
701 for (i = 0; i < L2_SIZE; ++i) {
702 pd[i].first_tb = NULL;
703 invalidate_page_bitmap(pd + i);
705 } else {
706 void **pp = *lp;
707 for (i = 0; i < L2_SIZE; ++i) {
708 page_flush_tb_1 (level - 1, pp + i);
713 static void page_flush_tb(void)
715 int i;
716 for (i = 0; i < V_L1_SIZE; i++) {
717 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
721 /* flush all the translation blocks */
722 /* XXX: tb_flush is currently not thread safe */
723 void tb_flush(CPUState *env1)
725 CPUState *env;
726 #if defined(DEBUG_FLUSH)
727 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
728 (unsigned long)(code_gen_ptr - code_gen_buffer),
729 nb_tbs, nb_tbs > 0 ?
730 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
731 #endif
732 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
733 cpu_abort(env1, "Internal error: code buffer overflow\n");
735 nb_tbs = 0;
737 for(env = first_cpu; env != NULL; env = env->next_cpu) {
738 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
741 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
742 page_flush_tb();
744 code_gen_ptr = code_gen_buffer;
745 /* XXX: flush processor icache at this point if cache flush is
746 expensive */
747 tb_flush_count++;
750 #ifdef DEBUG_TB_CHECK
752 static void tb_invalidate_check(target_ulong address)
754 TranslationBlock *tb;
755 int i;
756 address &= TARGET_PAGE_MASK;
757 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
758 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
759 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
760 address >= tb->pc + tb->size)) {
761 printf("ERROR invalidate: address=" TARGET_FMT_lx
762 " PC=%08lx size=%04x\n",
763 address, (long)tb->pc, tb->size);
769 /* verify that all the pages have correct rights for code */
770 static void tb_page_check(void)
772 TranslationBlock *tb;
773 int i, flags1, flags2;
775 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
776 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
777 flags1 = page_get_flags(tb->pc);
778 flags2 = page_get_flags(tb->pc + tb->size - 1);
779 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
780 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
781 (long)tb->pc, tb->size, flags1, flags2);
787 #endif
789 /* invalidate one TB */
790 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
791 int next_offset)
793 TranslationBlock *tb1;
794 for(;;) {
795 tb1 = *ptb;
796 if (tb1 == tb) {
797 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
798 break;
800 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
804 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
806 TranslationBlock *tb1;
807 unsigned int n1;
809 for(;;) {
810 tb1 = *ptb;
811 n1 = (long)tb1 & 3;
812 tb1 = (TranslationBlock *)((long)tb1 & ~3);
813 if (tb1 == tb) {
814 *ptb = tb1->page_next[n1];
815 break;
817 ptb = &tb1->page_next[n1];
821 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
823 TranslationBlock *tb1, **ptb;
824 unsigned int n1;
826 ptb = &tb->jmp_next[n];
827 tb1 = *ptb;
828 if (tb1) {
829 /* find tb(n) in circular list */
830 for(;;) {
831 tb1 = *ptb;
832 n1 = (long)tb1 & 3;
833 tb1 = (TranslationBlock *)((long)tb1 & ~3);
834 if (n1 == n && tb1 == tb)
835 break;
836 if (n1 == 2) {
837 ptb = &tb1->jmp_first;
838 } else {
839 ptb = &tb1->jmp_next[n1];
842 /* now we can suppress tb(n) from the list */
843 *ptb = tb->jmp_next[n];
845 tb->jmp_next[n] = NULL;
849 /* reset the jump entry 'n' of a TB so that it is not chained to
850 another TB */
851 static inline void tb_reset_jump(TranslationBlock *tb, int n)
853 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
856 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
858 CPUState *env;
859 PageDesc *p;
860 unsigned int h, n1;
861 tb_page_addr_t phys_pc;
862 TranslationBlock *tb1, *tb2;
864 /* remove the TB from the hash list */
865 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
866 h = tb_phys_hash_func(phys_pc);
867 tb_remove(&tb_phys_hash[h], tb,
868 offsetof(TranslationBlock, phys_hash_next));
870 /* remove the TB from the page list */
871 if (tb->page_addr[0] != page_addr) {
872 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
873 tb_page_remove(&p->first_tb, tb);
874 invalidate_page_bitmap(p);
876 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
877 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
878 tb_page_remove(&p->first_tb, tb);
879 invalidate_page_bitmap(p);
882 tb_invalidated_flag = 1;
884 /* remove the TB from the hash list */
885 h = tb_jmp_cache_hash_func(tb->pc);
886 for(env = first_cpu; env != NULL; env = env->next_cpu) {
887 if (env->tb_jmp_cache[h] == tb)
888 env->tb_jmp_cache[h] = NULL;
891 /* suppress this TB from the two jump lists */
892 tb_jmp_remove(tb, 0);
893 tb_jmp_remove(tb, 1);
895 /* suppress any remaining jumps to this TB */
896 tb1 = tb->jmp_first;
897 for(;;) {
898 n1 = (long)tb1 & 3;
899 if (n1 == 2)
900 break;
901 tb1 = (TranslationBlock *)((long)tb1 & ~3);
902 tb2 = tb1->jmp_next[n1];
903 tb_reset_jump(tb1, n1);
904 tb1->jmp_next[n1] = NULL;
905 tb1 = tb2;
907 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
909 tb_phys_invalidate_count++;
912 static inline void set_bits(uint8_t *tab, int start, int len)
914 int end, mask, end1;
916 end = start + len;
917 tab += start >> 3;
918 mask = 0xff << (start & 7);
919 if ((start & ~7) == (end & ~7)) {
920 if (start < end) {
921 mask &= ~(0xff << (end & 7));
922 *tab |= mask;
924 } else {
925 *tab++ |= mask;
926 start = (start + 8) & ~7;
927 end1 = end & ~7;
928 while (start < end1) {
929 *tab++ = 0xff;
930 start += 8;
932 if (start < end) {
933 mask = ~(0xff << (end & 7));
934 *tab |= mask;
939 static void build_page_bitmap(PageDesc *p)
941 int n, tb_start, tb_end;
942 TranslationBlock *tb;
944 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
946 tb = p->first_tb;
947 while (tb != NULL) {
948 n = (long)tb & 3;
949 tb = (TranslationBlock *)((long)tb & ~3);
950 /* NOTE: this is subtle as a TB may span two physical pages */
951 if (n == 0) {
952 /* NOTE: tb_end may be after the end of the page, but
953 it is not a problem */
954 tb_start = tb->pc & ~TARGET_PAGE_MASK;
955 tb_end = tb_start + tb->size;
956 if (tb_end > TARGET_PAGE_SIZE)
957 tb_end = TARGET_PAGE_SIZE;
958 } else {
959 tb_start = 0;
960 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
962 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
963 tb = tb->page_next[n];
967 TranslationBlock *tb_gen_code(CPUState *env,
968 target_ulong pc, target_ulong cs_base,
969 int flags, int cflags)
971 TranslationBlock *tb;
972 uint8_t *tc_ptr;
973 tb_page_addr_t phys_pc, phys_page2;
974 target_ulong virt_page2;
975 int code_gen_size;
977 phys_pc = get_page_addr_code(env, pc);
978 tb = tb_alloc(pc);
979 if (!tb) {
980 /* flush must be done */
981 tb_flush(env);
982 /* cannot fail at this point */
983 tb = tb_alloc(pc);
984 /* Don't forget to invalidate previous TB info. */
985 tb_invalidated_flag = 1;
987 tc_ptr = code_gen_ptr;
988 tb->tc_ptr = tc_ptr;
989 tb->cs_base = cs_base;
990 tb->flags = flags;
991 tb->cflags = cflags;
992 cpu_gen_code(env, tb, &code_gen_size);
993 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
995 /* check next page if needed */
996 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
997 phys_page2 = -1;
998 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
999 phys_page2 = get_page_addr_code(env, virt_page2);
1001 tb_link_page(tb, phys_pc, phys_page2);
1002 return tb;
1005 /* invalidate all TBs which intersect with the target physical page
1006 starting in range [start;end[. NOTE: start and end must refer to
1007 the same physical page. 'is_cpu_write_access' should be true if called
1008 from a real cpu write access: the virtual CPU will exit the current
1009 TB if code is modified inside this TB. */
1010 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1011 int is_cpu_write_access)
1013 TranslationBlock *tb, *tb_next, *saved_tb;
1014 CPUState *env = cpu_single_env;
1015 tb_page_addr_t tb_start, tb_end;
1016 PageDesc *p;
1017 int n;
1018 #ifdef TARGET_HAS_PRECISE_SMC
1019 int current_tb_not_found = is_cpu_write_access;
1020 TranslationBlock *current_tb = NULL;
1021 int current_tb_modified = 0;
1022 target_ulong current_pc = 0;
1023 target_ulong current_cs_base = 0;
1024 int current_flags = 0;
1025 #endif /* TARGET_HAS_PRECISE_SMC */
1027 p = page_find(start >> TARGET_PAGE_BITS);
1028 if (!p)
1029 return;
1030 if (!p->code_bitmap &&
1031 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1032 is_cpu_write_access) {
1033 /* build code bitmap */
1034 build_page_bitmap(p);
1037 /* we remove all the TBs in the range [start, end[ */
1038 /* XXX: see if in some cases it could be faster to invalidate all the code */
1039 tb = p->first_tb;
1040 while (tb != NULL) {
1041 n = (long)tb & 3;
1042 tb = (TranslationBlock *)((long)tb & ~3);
1043 tb_next = tb->page_next[n];
1044 /* NOTE: this is subtle as a TB may span two physical pages */
1045 if (n == 0) {
1046 /* NOTE: tb_end may be after the end of the page, but
1047 it is not a problem */
1048 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1049 tb_end = tb_start + tb->size;
1050 } else {
1051 tb_start = tb->page_addr[1];
1052 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1054 if (!(tb_end <= start || tb_start >= end)) {
1055 #ifdef TARGET_HAS_PRECISE_SMC
1056 if (current_tb_not_found) {
1057 current_tb_not_found = 0;
1058 current_tb = NULL;
1059 if (env->mem_io_pc) {
1060 /* now we have a real cpu fault */
1061 current_tb = tb_find_pc(env->mem_io_pc);
1064 if (current_tb == tb &&
1065 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1066 /* If we are modifying the current TB, we must stop
1067 its execution. We could be more precise by checking
1068 that the modification is after the current PC, but it
1069 would require a specialized function to partially
1070 restore the CPU state */
1072 current_tb_modified = 1;
1073 cpu_restore_state(current_tb, env, env->mem_io_pc);
1074 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1075 &current_flags);
1077 #endif /* TARGET_HAS_PRECISE_SMC */
1078 /* we need to do that to handle the case where a signal
1079 occurs while doing tb_phys_invalidate() */
1080 saved_tb = NULL;
1081 if (env) {
1082 saved_tb = env->current_tb;
1083 env->current_tb = NULL;
1085 tb_phys_invalidate(tb, -1);
1086 if (env) {
1087 env->current_tb = saved_tb;
1088 if (env->interrupt_request && env->current_tb)
1089 cpu_interrupt(env, env->interrupt_request);
1092 tb = tb_next;
1094 #if !defined(CONFIG_USER_ONLY)
1095 /* if no code remaining, no need to continue to use slow writes */
1096 if (!p->first_tb) {
1097 invalidate_page_bitmap(p);
1098 if (is_cpu_write_access) {
1099 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1102 #endif
1103 #ifdef TARGET_HAS_PRECISE_SMC
1104 if (current_tb_modified) {
1105 /* we generate a block containing just the instruction
1106 modifying the memory. It will ensure that it cannot modify
1107 itself */
1108 env->current_tb = NULL;
1109 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1110 cpu_resume_from_signal(env, NULL);
1112 #endif
1115 /* len must be <= 8 and start must be a multiple of len */
1116 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1118 PageDesc *p;
1119 int offset, b;
1120 #if 0
1121 if (1) {
1122 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1123 cpu_single_env->mem_io_vaddr, len,
1124 cpu_single_env->eip,
1125 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1127 #endif
1128 p = page_find(start >> TARGET_PAGE_BITS);
1129 if (!p)
1130 return;
1131 if (p->code_bitmap) {
1132 offset = start & ~TARGET_PAGE_MASK;
1133 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1134 if (b & ((1 << len) - 1))
1135 goto do_invalidate;
1136 } else {
1137 do_invalidate:
1138 tb_invalidate_phys_page_range(start, start + len, 1);
1142 #if !defined(CONFIG_SOFTMMU)
1143 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1144 unsigned long pc, void *puc)
1146 TranslationBlock *tb;
1147 PageDesc *p;
1148 int n;
1149 #ifdef TARGET_HAS_PRECISE_SMC
1150 TranslationBlock *current_tb = NULL;
1151 CPUState *env = cpu_single_env;
1152 int current_tb_modified = 0;
1153 target_ulong current_pc = 0;
1154 target_ulong current_cs_base = 0;
1155 int current_flags = 0;
1156 #endif
1158 addr &= TARGET_PAGE_MASK;
1159 p = page_find(addr >> TARGET_PAGE_BITS);
1160 if (!p)
1161 return;
1162 tb = p->first_tb;
1163 #ifdef TARGET_HAS_PRECISE_SMC
1164 if (tb && pc != 0) {
1165 current_tb = tb_find_pc(pc);
1167 #endif
1168 while (tb != NULL) {
1169 n = (long)tb & 3;
1170 tb = (TranslationBlock *)((long)tb & ~3);
1171 #ifdef TARGET_HAS_PRECISE_SMC
1172 if (current_tb == tb &&
1173 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1174 /* If we are modifying the current TB, we must stop
1175 its execution. We could be more precise by checking
1176 that the modification is after the current PC, but it
1177 would require a specialized function to partially
1178 restore the CPU state */
1180 current_tb_modified = 1;
1181 cpu_restore_state(current_tb, env, pc);
1182 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1183 &current_flags);
1185 #endif /* TARGET_HAS_PRECISE_SMC */
1186 tb_phys_invalidate(tb, addr);
1187 tb = tb->page_next[n];
1189 p->first_tb = NULL;
1190 #ifdef TARGET_HAS_PRECISE_SMC
1191 if (current_tb_modified) {
1192 /* we generate a block containing just the instruction
1193 modifying the memory. It will ensure that it cannot modify
1194 itself */
1195 env->current_tb = NULL;
1196 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1197 cpu_resume_from_signal(env, puc);
1199 #endif
1201 #endif
1203 /* add the tb in the target page and protect it if necessary */
1204 static inline void tb_alloc_page(TranslationBlock *tb,
1205 unsigned int n, tb_page_addr_t page_addr)
1207 PageDesc *p;
1208 TranslationBlock *last_first_tb;
1210 tb->page_addr[n] = page_addr;
1211 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1212 tb->page_next[n] = p->first_tb;
1213 last_first_tb = p->first_tb;
1214 p->first_tb = (TranslationBlock *)((long)tb | n);
1215 invalidate_page_bitmap(p);
1217 #if defined(TARGET_HAS_SMC) || 1
1219 #if defined(CONFIG_USER_ONLY)
1220 if (p->flags & PAGE_WRITE) {
1221 target_ulong addr;
1222 PageDesc *p2;
1223 int prot;
1225 /* force the host page as non writable (writes will have a
1226 page fault + mprotect overhead) */
1227 page_addr &= qemu_host_page_mask;
1228 prot = 0;
1229 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1230 addr += TARGET_PAGE_SIZE) {
1232 p2 = page_find (addr >> TARGET_PAGE_BITS);
1233 if (!p2)
1234 continue;
1235 prot |= p2->flags;
1236 p2->flags &= ~PAGE_WRITE;
1238 mprotect(g2h(page_addr), qemu_host_page_size,
1239 (prot & PAGE_BITS) & ~PAGE_WRITE);
1240 #ifdef DEBUG_TB_INVALIDATE
1241 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1242 page_addr);
1243 #endif
1245 #else
1246 /* if some code is already present, then the pages are already
1247 protected. So we handle the case where only the first TB is
1248 allocated in a physical page */
1249 if (!last_first_tb) {
1250 tlb_protect_code(page_addr);
1252 #endif
1254 #endif /* TARGET_HAS_SMC */
1257 /* add a new TB and link it to the physical page tables. phys_page2 is
1258 (-1) to indicate that only one page contains the TB. */
1259 void tb_link_page(TranslationBlock *tb,
1260 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1262 unsigned int h;
1263 TranslationBlock **ptb;
1265 /* Grab the mmap lock to stop another thread invalidating this TB
1266 before we are done. */
1267 mmap_lock();
1268 /* add in the physical hash table */
1269 h = tb_phys_hash_func(phys_pc);
1270 ptb = &tb_phys_hash[h];
1271 tb->phys_hash_next = *ptb;
1272 *ptb = tb;
1274 /* add in the page list */
1275 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1276 if (phys_page2 != -1)
1277 tb_alloc_page(tb, 1, phys_page2);
1278 else
1279 tb->page_addr[1] = -1;
1281 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1282 tb->jmp_next[0] = NULL;
1283 tb->jmp_next[1] = NULL;
1285 /* init original jump addresses */
1286 if (tb->tb_next_offset[0] != 0xffff)
1287 tb_reset_jump(tb, 0);
1288 if (tb->tb_next_offset[1] != 0xffff)
1289 tb_reset_jump(tb, 1);
1291 #ifdef DEBUG_TB_CHECK
1292 tb_page_check();
1293 #endif
1294 mmap_unlock();
1297 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1298 tb[1].tc_ptr. Return NULL if not found */
1299 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1301 int m_min, m_max, m;
1302 unsigned long v;
1303 TranslationBlock *tb;
1305 if (nb_tbs <= 0)
1306 return NULL;
1307 if (tc_ptr < (unsigned long)code_gen_buffer ||
1308 tc_ptr >= (unsigned long)code_gen_ptr)
1309 return NULL;
1310 /* binary search (cf Knuth) */
1311 m_min = 0;
1312 m_max = nb_tbs - 1;
1313 while (m_min <= m_max) {
1314 m = (m_min + m_max) >> 1;
1315 tb = &tbs[m];
1316 v = (unsigned long)tb->tc_ptr;
1317 if (v == tc_ptr)
1318 return tb;
1319 else if (tc_ptr < v) {
1320 m_max = m - 1;
1321 } else {
1322 m_min = m + 1;
1325 return &tbs[m_max];
1328 static void tb_reset_jump_recursive(TranslationBlock *tb);
1330 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1332 TranslationBlock *tb1, *tb_next, **ptb;
1333 unsigned int n1;
1335 tb1 = tb->jmp_next[n];
1336 if (tb1 != NULL) {
1337 /* find head of list */
1338 for(;;) {
1339 n1 = (long)tb1 & 3;
1340 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1341 if (n1 == 2)
1342 break;
1343 tb1 = tb1->jmp_next[n1];
1345 /* we are now sure now that tb jumps to tb1 */
1346 tb_next = tb1;
1348 /* remove tb from the jmp_first list */
1349 ptb = &tb_next->jmp_first;
1350 for(;;) {
1351 tb1 = *ptb;
1352 n1 = (long)tb1 & 3;
1353 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1354 if (n1 == n && tb1 == tb)
1355 break;
1356 ptb = &tb1->jmp_next[n1];
1358 *ptb = tb->jmp_next[n];
1359 tb->jmp_next[n] = NULL;
1361 /* suppress the jump to next tb in generated code */
1362 tb_reset_jump(tb, n);
1364 /* suppress jumps in the tb on which we could have jumped */
1365 tb_reset_jump_recursive(tb_next);
1369 static void tb_reset_jump_recursive(TranslationBlock *tb)
1371 tb_reset_jump_recursive2(tb, 0);
1372 tb_reset_jump_recursive2(tb, 1);
1375 #if defined(TARGET_HAS_ICE)
1376 #if defined(CONFIG_USER_ONLY)
1377 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1379 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1381 #else
1382 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1384 target_phys_addr_t addr;
1385 target_ulong pd;
1386 ram_addr_t ram_addr;
1387 PhysPageDesc *p;
1389 addr = cpu_get_phys_page_debug(env, pc);
1390 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1391 if (!p) {
1392 pd = IO_MEM_UNASSIGNED;
1393 } else {
1394 pd = p->phys_offset;
1396 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1397 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1399 #endif
1400 #endif /* TARGET_HAS_ICE */
1402 #if defined(CONFIG_USER_ONLY)
1403 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1408 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1409 int flags, CPUWatchpoint **watchpoint)
1411 return -ENOSYS;
1413 #else
1414 /* Add a watchpoint. */
1415 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1416 int flags, CPUWatchpoint **watchpoint)
1418 target_ulong len_mask = ~(len - 1);
1419 CPUWatchpoint *wp;
1421 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1422 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1423 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1424 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1425 return -EINVAL;
1427 wp = qemu_malloc(sizeof(*wp));
1429 wp->vaddr = addr;
1430 wp->len_mask = len_mask;
1431 wp->flags = flags;
1433 /* keep all GDB-injected watchpoints in front */
1434 if (flags & BP_GDB)
1435 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1436 else
1437 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1439 tlb_flush_page(env, addr);
1441 if (watchpoint)
1442 *watchpoint = wp;
1443 return 0;
1446 /* Remove a specific watchpoint. */
1447 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1448 int flags)
1450 target_ulong len_mask = ~(len - 1);
1451 CPUWatchpoint *wp;
1453 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1454 if (addr == wp->vaddr && len_mask == wp->len_mask
1455 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1456 cpu_watchpoint_remove_by_ref(env, wp);
1457 return 0;
1460 return -ENOENT;
1463 /* Remove a specific watchpoint by reference. */
1464 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1466 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1468 tlb_flush_page(env, watchpoint->vaddr);
1470 qemu_free(watchpoint);
1473 /* Remove all matching watchpoints. */
1474 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1476 CPUWatchpoint *wp, *next;
1478 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1479 if (wp->flags & mask)
1480 cpu_watchpoint_remove_by_ref(env, wp);
1483 #endif
1485 /* Add a breakpoint. */
1486 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1487 CPUBreakpoint **breakpoint)
1489 #if defined(TARGET_HAS_ICE)
1490 CPUBreakpoint *bp;
1492 bp = qemu_malloc(sizeof(*bp));
1494 bp->pc = pc;
1495 bp->flags = flags;
1497 /* keep all GDB-injected breakpoints in front */
1498 if (flags & BP_GDB)
1499 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1500 else
1501 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1503 breakpoint_invalidate(env, pc);
1505 if (breakpoint)
1506 *breakpoint = bp;
1507 return 0;
1508 #else
1509 return -ENOSYS;
1510 #endif
1513 /* Remove a specific breakpoint. */
1514 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1516 #if defined(TARGET_HAS_ICE)
1517 CPUBreakpoint *bp;
1519 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1520 if (bp->pc == pc && bp->flags == flags) {
1521 cpu_breakpoint_remove_by_ref(env, bp);
1522 return 0;
1525 return -ENOENT;
1526 #else
1527 return -ENOSYS;
1528 #endif
1531 /* Remove a specific breakpoint by reference. */
1532 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1534 #if defined(TARGET_HAS_ICE)
1535 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1537 breakpoint_invalidate(env, breakpoint->pc);
1539 qemu_free(breakpoint);
1540 #endif
1543 /* Remove all matching breakpoints. */
1544 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1546 #if defined(TARGET_HAS_ICE)
1547 CPUBreakpoint *bp, *next;
1549 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1550 if (bp->flags & mask)
1551 cpu_breakpoint_remove_by_ref(env, bp);
1553 #endif
1556 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1557 CPU loop after each instruction */
1558 void cpu_single_step(CPUState *env, int enabled)
1560 #if defined(TARGET_HAS_ICE)
1561 if (env->singlestep_enabled != enabled) {
1562 env->singlestep_enabled = enabled;
1563 if (kvm_enabled())
1564 kvm_update_guest_debug(env, 0);
1565 else {
1566 /* must flush all the translated code to avoid inconsistencies */
1567 /* XXX: only flush what is necessary */
1568 tb_flush(env);
1571 #endif
1574 /* enable or disable low levels log */
1575 void cpu_set_log(int log_flags)
1577 loglevel = log_flags;
1578 if (loglevel && !logfile) {
1579 logfile = fopen(logfilename, log_append ? "a" : "w");
1580 if (!logfile) {
1581 perror(logfilename);
1582 _exit(1);
1584 #if !defined(CONFIG_SOFTMMU)
1585 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1587 static char logfile_buf[4096];
1588 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1590 #elif !defined(_WIN32)
1591 /* Win32 doesn't support line-buffering and requires size >= 2 */
1592 setvbuf(logfile, NULL, _IOLBF, 0);
1593 #endif
1594 log_append = 1;
1596 if (!loglevel && logfile) {
1597 fclose(logfile);
1598 logfile = NULL;
1602 void cpu_set_log_filename(const char *filename)
1604 logfilename = strdup(filename);
1605 if (logfile) {
1606 fclose(logfile);
1607 logfile = NULL;
1609 cpu_set_log(loglevel);
1612 static void cpu_unlink_tb(CPUState *env)
1614 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1615 problem and hope the cpu will stop of its own accord. For userspace
1616 emulation this often isn't actually as bad as it sounds. Often
1617 signals are used primarily to interrupt blocking syscalls. */
1618 TranslationBlock *tb;
1619 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1621 spin_lock(&interrupt_lock);
1622 tb = env->current_tb;
1623 /* if the cpu is currently executing code, we must unlink it and
1624 all the potentially executing TB */
1625 if (tb) {
1626 env->current_tb = NULL;
1627 tb_reset_jump_recursive(tb);
1629 spin_unlock(&interrupt_lock);
1632 /* mask must never be zero, except for A20 change call */
1633 void cpu_interrupt(CPUState *env, int mask)
1635 int old_mask;
1637 old_mask = env->interrupt_request;
1638 env->interrupt_request |= mask;
1640 #ifndef CONFIG_USER_ONLY
1642 * If called from iothread context, wake the target cpu in
1643 * case its halted.
1645 if (!qemu_cpu_is_self(env)) {
1646 qemu_cpu_kick(env);
1647 return;
1649 #endif
1651 if (use_icount) {
1652 env->icount_decr.u16.high = 0xffff;
1653 #ifndef CONFIG_USER_ONLY
1654 if (!can_do_io(env)
1655 && (mask & ~old_mask) != 0) {
1656 cpu_abort(env, "Raised interrupt while not in I/O function");
1658 #endif
1659 } else {
1660 cpu_unlink_tb(env);
1664 void cpu_reset_interrupt(CPUState *env, int mask)
1666 env->interrupt_request &= ~mask;
1669 void cpu_exit(CPUState *env)
1671 env->exit_request = 1;
1672 cpu_unlink_tb(env);
1675 const CPULogItem cpu_log_items[] = {
1676 { CPU_LOG_TB_OUT_ASM, "out_asm",
1677 "show generated host assembly code for each compiled TB" },
1678 { CPU_LOG_TB_IN_ASM, "in_asm",
1679 "show target assembly code for each compiled TB" },
1680 { CPU_LOG_TB_OP, "op",
1681 "show micro ops for each compiled TB" },
1682 { CPU_LOG_TB_OP_OPT, "op_opt",
1683 "show micro ops "
1684 #ifdef TARGET_I386
1685 "before eflags optimization and "
1686 #endif
1687 "after liveness analysis" },
1688 { CPU_LOG_INT, "int",
1689 "show interrupts/exceptions in short format" },
1690 { CPU_LOG_EXEC, "exec",
1691 "show trace before each executed TB (lots of logs)" },
1692 { CPU_LOG_TB_CPU, "cpu",
1693 "show CPU state before block translation" },
1694 #ifdef TARGET_I386
1695 { CPU_LOG_PCALL, "pcall",
1696 "show protected mode far calls/returns/exceptions" },
1697 { CPU_LOG_RESET, "cpu_reset",
1698 "show CPU state before CPU resets" },
1699 #endif
1700 #ifdef DEBUG_IOPORT
1701 { CPU_LOG_IOPORT, "ioport",
1702 "show all i/o ports accesses" },
1703 #endif
1704 { 0, NULL, NULL },
1707 #ifndef CONFIG_USER_ONLY
1708 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1709 = QLIST_HEAD_INITIALIZER(memory_client_list);
1711 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1712 ram_addr_t size,
1713 ram_addr_t phys_offset)
1715 CPUPhysMemoryClient *client;
1716 QLIST_FOREACH(client, &memory_client_list, list) {
1717 client->set_memory(client, start_addr, size, phys_offset);
1721 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1722 target_phys_addr_t end)
1724 CPUPhysMemoryClient *client;
1725 QLIST_FOREACH(client, &memory_client_list, list) {
1726 int r = client->sync_dirty_bitmap(client, start, end);
1727 if (r < 0)
1728 return r;
1730 return 0;
1733 static int cpu_notify_migration_log(int enable)
1735 CPUPhysMemoryClient *client;
1736 QLIST_FOREACH(client, &memory_client_list, list) {
1737 int r = client->migration_log(client, enable);
1738 if (r < 0)
1739 return r;
1741 return 0;
1744 static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1745 int level, void **lp)
1747 int i;
1749 if (*lp == NULL) {
1750 return;
1752 if (level == 0) {
1753 PhysPageDesc *pd = *lp;
1754 for (i = 0; i < L2_SIZE; ++i) {
1755 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1756 client->set_memory(client, pd[i].region_offset,
1757 TARGET_PAGE_SIZE, pd[i].phys_offset);
1760 } else {
1761 void **pp = *lp;
1762 for (i = 0; i < L2_SIZE; ++i) {
1763 phys_page_for_each_1(client, level - 1, pp + i);
1768 static void phys_page_for_each(CPUPhysMemoryClient *client)
1770 int i;
1771 for (i = 0; i < P_L1_SIZE; ++i) {
1772 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1773 l1_phys_map + 1);
1777 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1779 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1780 phys_page_for_each(client);
1783 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1785 QLIST_REMOVE(client, list);
1787 #endif
1789 static int cmp1(const char *s1, int n, const char *s2)
1791 if (strlen(s2) != n)
1792 return 0;
1793 return memcmp(s1, s2, n) == 0;
1796 /* takes a comma separated list of log masks. Return 0 if error. */
1797 int cpu_str_to_log_mask(const char *str)
1799 const CPULogItem *item;
1800 int mask;
1801 const char *p, *p1;
1803 p = str;
1804 mask = 0;
1805 for(;;) {
1806 p1 = strchr(p, ',');
1807 if (!p1)
1808 p1 = p + strlen(p);
1809 if(cmp1(p,p1-p,"all")) {
1810 for(item = cpu_log_items; item->mask != 0; item++) {
1811 mask |= item->mask;
1813 } else {
1814 for(item = cpu_log_items; item->mask != 0; item++) {
1815 if (cmp1(p, p1 - p, item->name))
1816 goto found;
1818 return 0;
1820 found:
1821 mask |= item->mask;
1822 if (*p1 != ',')
1823 break;
1824 p = p1 + 1;
1826 return mask;
1829 void cpu_abort(CPUState *env, const char *fmt, ...)
1831 va_list ap;
1832 va_list ap2;
1834 va_start(ap, fmt);
1835 va_copy(ap2, ap);
1836 fprintf(stderr, "qemu: fatal: ");
1837 vfprintf(stderr, fmt, ap);
1838 fprintf(stderr, "\n");
1839 #ifdef TARGET_I386
1840 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1841 #else
1842 cpu_dump_state(env, stderr, fprintf, 0);
1843 #endif
1844 if (qemu_log_enabled()) {
1845 qemu_log("qemu: fatal: ");
1846 qemu_log_vprintf(fmt, ap2);
1847 qemu_log("\n");
1848 #ifdef TARGET_I386
1849 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1850 #else
1851 log_cpu_state(env, 0);
1852 #endif
1853 qemu_log_flush();
1854 qemu_log_close();
1856 va_end(ap2);
1857 va_end(ap);
1858 #if defined(CONFIG_USER_ONLY)
1860 struct sigaction act;
1861 sigfillset(&act.sa_mask);
1862 act.sa_handler = SIG_DFL;
1863 sigaction(SIGABRT, &act, NULL);
1865 #endif
1866 abort();
1869 CPUState *cpu_copy(CPUState *env)
1871 CPUState *new_env = cpu_init(env->cpu_model_str);
1872 CPUState *next_cpu = new_env->next_cpu;
1873 int cpu_index = new_env->cpu_index;
1874 #if defined(TARGET_HAS_ICE)
1875 CPUBreakpoint *bp;
1876 CPUWatchpoint *wp;
1877 #endif
1879 memcpy(new_env, env, sizeof(CPUState));
1881 /* Preserve chaining and index. */
1882 new_env->next_cpu = next_cpu;
1883 new_env->cpu_index = cpu_index;
1885 /* Clone all break/watchpoints.
1886 Note: Once we support ptrace with hw-debug register access, make sure
1887 BP_CPU break/watchpoints are handled correctly on clone. */
1888 QTAILQ_INIT(&env->breakpoints);
1889 QTAILQ_INIT(&env->watchpoints);
1890 #if defined(TARGET_HAS_ICE)
1891 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1892 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1894 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1895 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1896 wp->flags, NULL);
1898 #endif
1900 return new_env;
1903 #if !defined(CONFIG_USER_ONLY)
1905 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1907 unsigned int i;
1909 /* Discard jump cache entries for any tb which might potentially
1910 overlap the flushed page. */
1911 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1912 memset (&env->tb_jmp_cache[i], 0,
1913 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1915 i = tb_jmp_cache_hash_page(addr);
1916 memset (&env->tb_jmp_cache[i], 0,
1917 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1920 static CPUTLBEntry s_cputlb_empty_entry = {
1921 .addr_read = -1,
1922 .addr_write = -1,
1923 .addr_code = -1,
1924 .addend = -1,
1927 /* NOTE: if flush_global is true, also flush global entries (not
1928 implemented yet) */
1929 void tlb_flush(CPUState *env, int flush_global)
1931 int i;
1933 #if defined(DEBUG_TLB)
1934 printf("tlb_flush:\n");
1935 #endif
1936 /* must reset current TB so that interrupts cannot modify the
1937 links while we are modifying them */
1938 env->current_tb = NULL;
1940 for(i = 0; i < CPU_TLB_SIZE; i++) {
1941 int mmu_idx;
1942 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1943 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1947 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1949 env->tlb_flush_addr = -1;
1950 env->tlb_flush_mask = 0;
1951 tlb_flush_count++;
1954 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1956 if (addr == (tlb_entry->addr_read &
1957 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1958 addr == (tlb_entry->addr_write &
1959 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1960 addr == (tlb_entry->addr_code &
1961 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1962 *tlb_entry = s_cputlb_empty_entry;
1966 void tlb_flush_page(CPUState *env, target_ulong addr)
1968 int i;
1969 int mmu_idx;
1971 #if defined(DEBUG_TLB)
1972 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1973 #endif
1974 /* Check if we need to flush due to large pages. */
1975 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1976 #if defined(DEBUG_TLB)
1977 printf("tlb_flush_page: forced full flush ("
1978 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1979 env->tlb_flush_addr, env->tlb_flush_mask);
1980 #endif
1981 tlb_flush(env, 1);
1982 return;
1984 /* must reset current TB so that interrupts cannot modify the
1985 links while we are modifying them */
1986 env->current_tb = NULL;
1988 addr &= TARGET_PAGE_MASK;
1989 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1990 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1991 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1993 tlb_flush_jmp_cache(env, addr);
1996 /* update the TLBs so that writes to code in the virtual page 'addr'
1997 can be detected */
1998 static void tlb_protect_code(ram_addr_t ram_addr)
2000 cpu_physical_memory_reset_dirty(ram_addr,
2001 ram_addr + TARGET_PAGE_SIZE,
2002 CODE_DIRTY_FLAG);
2005 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2006 tested for self modifying code */
2007 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2008 target_ulong vaddr)
2010 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2013 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2014 unsigned long start, unsigned long length)
2016 unsigned long addr;
2017 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2018 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2019 if ((addr - start) < length) {
2020 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2025 /* Note: start and end must be within the same ram block. */
2026 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2027 int dirty_flags)
2029 CPUState *env;
2030 unsigned long length, start1;
2031 int i;
2033 start &= TARGET_PAGE_MASK;
2034 end = TARGET_PAGE_ALIGN(end);
2036 length = end - start;
2037 if (length == 0)
2038 return;
2039 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2041 /* we modify the TLB cache so that the dirty bit will be set again
2042 when accessing the range */
2043 start1 = (unsigned long)qemu_safe_ram_ptr(start);
2044 /* Chek that we don't span multiple blocks - this breaks the
2045 address comparisons below. */
2046 if ((unsigned long)qemu_safe_ram_ptr(end - 1) - start1
2047 != (end - 1) - start) {
2048 abort();
2051 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2052 int mmu_idx;
2053 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2054 for(i = 0; i < CPU_TLB_SIZE; i++)
2055 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2056 start1, length);
2061 int cpu_physical_memory_set_dirty_tracking(int enable)
2063 int ret = 0;
2064 in_migration = enable;
2065 ret = cpu_notify_migration_log(!!enable);
2066 return ret;
2069 int cpu_physical_memory_get_dirty_tracking(void)
2071 return in_migration;
2074 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2075 target_phys_addr_t end_addr)
2077 int ret;
2079 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2080 return ret;
2083 int cpu_physical_log_start(target_phys_addr_t start_addr,
2084 ram_addr_t size)
2086 CPUPhysMemoryClient *client;
2087 QLIST_FOREACH(client, &memory_client_list, list) {
2088 if (client->log_start) {
2089 int r = client->log_start(client, start_addr, size);
2090 if (r < 0) {
2091 return r;
2095 return 0;
2098 int cpu_physical_log_stop(target_phys_addr_t start_addr,
2099 ram_addr_t size)
2101 CPUPhysMemoryClient *client;
2102 QLIST_FOREACH(client, &memory_client_list, list) {
2103 if (client->log_stop) {
2104 int r = client->log_stop(client, start_addr, size);
2105 if (r < 0) {
2106 return r;
2110 return 0;
2113 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2115 ram_addr_t ram_addr;
2116 void *p;
2118 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2119 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2120 + tlb_entry->addend);
2121 ram_addr = qemu_ram_addr_from_host_nofail(p);
2122 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2123 tlb_entry->addr_write |= TLB_NOTDIRTY;
2128 /* update the TLB according to the current state of the dirty bits */
2129 void cpu_tlb_update_dirty(CPUState *env)
2131 int i;
2132 int mmu_idx;
2133 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2134 for(i = 0; i < CPU_TLB_SIZE; i++)
2135 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2139 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2141 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2142 tlb_entry->addr_write = vaddr;
2145 /* update the TLB corresponding to virtual page vaddr
2146 so that it is no longer dirty */
2147 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2149 int i;
2150 int mmu_idx;
2152 vaddr &= TARGET_PAGE_MASK;
2153 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2154 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2155 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2158 /* Our TLB does not support large pages, so remember the area covered by
2159 large pages and trigger a full TLB flush if these are invalidated. */
2160 static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2161 target_ulong size)
2163 target_ulong mask = ~(size - 1);
2165 if (env->tlb_flush_addr == (target_ulong)-1) {
2166 env->tlb_flush_addr = vaddr & mask;
2167 env->tlb_flush_mask = mask;
2168 return;
2170 /* Extend the existing region to include the new page.
2171 This is a compromise between unnecessary flushes and the cost
2172 of maintaining a full variable size TLB. */
2173 mask &= env->tlb_flush_mask;
2174 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2175 mask <<= 1;
2177 env->tlb_flush_addr &= mask;
2178 env->tlb_flush_mask = mask;
2181 /* Add a new TLB entry. At most one entry for a given virtual address
2182 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2183 supplied size is only used by tlb_flush_page. */
2184 void tlb_set_page(CPUState *env, target_ulong vaddr,
2185 target_phys_addr_t paddr, int prot,
2186 int mmu_idx, target_ulong size)
2188 PhysPageDesc *p;
2189 unsigned long pd;
2190 unsigned int index;
2191 target_ulong address;
2192 target_ulong code_address;
2193 unsigned long addend;
2194 CPUTLBEntry *te;
2195 CPUWatchpoint *wp;
2196 target_phys_addr_t iotlb;
2198 assert(size >= TARGET_PAGE_SIZE);
2199 if (size != TARGET_PAGE_SIZE) {
2200 tlb_add_large_page(env, vaddr, size);
2202 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2203 if (!p) {
2204 pd = IO_MEM_UNASSIGNED;
2205 } else {
2206 pd = p->phys_offset;
2208 #if defined(DEBUG_TLB)
2209 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
2210 " prot=%x idx=%d pd=0x%08lx\n",
2211 vaddr, paddr, prot, mmu_idx, pd);
2212 #endif
2214 address = vaddr;
2215 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2216 /* IO memory case (romd handled later) */
2217 address |= TLB_MMIO;
2219 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2220 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2221 /* Normal RAM. */
2222 iotlb = pd & TARGET_PAGE_MASK;
2223 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2224 iotlb |= IO_MEM_NOTDIRTY;
2225 else
2226 iotlb |= IO_MEM_ROM;
2227 } else {
2228 /* IO handlers are currently passed a physical address.
2229 It would be nice to pass an offset from the base address
2230 of that region. This would avoid having to special case RAM,
2231 and avoid full address decoding in every device.
2232 We can't use the high bits of pd for this because
2233 IO_MEM_ROMD uses these as a ram address. */
2234 iotlb = (pd & ~TARGET_PAGE_MASK);
2235 if (p) {
2236 iotlb += p->region_offset;
2237 } else {
2238 iotlb += paddr;
2242 code_address = address;
2243 /* Make accesses to pages with watchpoints go via the
2244 watchpoint trap routines. */
2245 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2246 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2247 /* Avoid trapping reads of pages with a write breakpoint. */
2248 if ((prot & PAGE_WRITE) || (wp->flags & BP_MEM_READ)) {
2249 iotlb = io_mem_watch + paddr;
2250 address |= TLB_MMIO;
2251 break;
2256 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2257 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2258 te = &env->tlb_table[mmu_idx][index];
2259 te->addend = addend - vaddr;
2260 if (prot & PAGE_READ) {
2261 te->addr_read = address;
2262 } else {
2263 te->addr_read = -1;
2266 if (prot & PAGE_EXEC) {
2267 te->addr_code = code_address;
2268 } else {
2269 te->addr_code = -1;
2271 if (prot & PAGE_WRITE) {
2272 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2273 (pd & IO_MEM_ROMD)) {
2274 /* Write access calls the I/O callback. */
2275 te->addr_write = address | TLB_MMIO;
2276 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2277 !cpu_physical_memory_is_dirty(pd)) {
2278 te->addr_write = address | TLB_NOTDIRTY;
2279 } else {
2280 te->addr_write = address;
2282 } else {
2283 te->addr_write = -1;
2287 #else
2289 void tlb_flush(CPUState *env, int flush_global)
2293 void tlb_flush_page(CPUState *env, target_ulong addr)
2298 * Walks guest process memory "regions" one by one
2299 * and calls callback function 'fn' for each region.
2302 struct walk_memory_regions_data
2304 walk_memory_regions_fn fn;
2305 void *priv;
2306 unsigned long start;
2307 int prot;
2310 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2311 abi_ulong end, int new_prot)
2313 if (data->start != -1ul) {
2314 int rc = data->fn(data->priv, data->start, end, data->prot);
2315 if (rc != 0) {
2316 return rc;
2320 data->start = (new_prot ? end : -1ul);
2321 data->prot = new_prot;
2323 return 0;
2326 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2327 abi_ulong base, int level, void **lp)
2329 abi_ulong pa;
2330 int i, rc;
2332 if (*lp == NULL) {
2333 return walk_memory_regions_end(data, base, 0);
2336 if (level == 0) {
2337 PageDesc *pd = *lp;
2338 for (i = 0; i < L2_SIZE; ++i) {
2339 int prot = pd[i].flags;
2341 pa = base | (i << TARGET_PAGE_BITS);
2342 if (prot != data->prot) {
2343 rc = walk_memory_regions_end(data, pa, prot);
2344 if (rc != 0) {
2345 return rc;
2349 } else {
2350 void **pp = *lp;
2351 for (i = 0; i < L2_SIZE; ++i) {
2352 pa = base | ((abi_ulong)i <<
2353 (TARGET_PAGE_BITS + L2_BITS * level));
2354 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2355 if (rc != 0) {
2356 return rc;
2361 return 0;
2364 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2366 struct walk_memory_regions_data data;
2367 unsigned long i;
2369 data.fn = fn;
2370 data.priv = priv;
2371 data.start = -1ul;
2372 data.prot = 0;
2374 for (i = 0; i < V_L1_SIZE; i++) {
2375 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2376 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2377 if (rc != 0) {
2378 return rc;
2382 return walk_memory_regions_end(&data, 0, 0);
2385 static int dump_region(void *priv, abi_ulong start,
2386 abi_ulong end, unsigned long prot)
2388 FILE *f = (FILE *)priv;
2390 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2391 " "TARGET_ABI_FMT_lx" %c%c%c\n",
2392 start, end, end - start,
2393 ((prot & PAGE_READ) ? 'r' : '-'),
2394 ((prot & PAGE_WRITE) ? 'w' : '-'),
2395 ((prot & PAGE_EXEC) ? 'x' : '-'));
2397 return (0);
2400 /* dump memory mappings */
2401 void page_dump(FILE *f)
2403 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2404 "start", "end", "size", "prot");
2405 walk_memory_regions(f, dump_region);
2408 int page_get_flags(target_ulong address)
2410 PageDesc *p;
2412 p = page_find(address >> TARGET_PAGE_BITS);
2413 if (!p)
2414 return 0;
2415 return p->flags;
2418 /* Modify the flags of a page and invalidate the code if necessary.
2419 The flag PAGE_WRITE_ORG is positioned automatically depending
2420 on PAGE_WRITE. The mmap_lock should already be held. */
2421 void page_set_flags(target_ulong start, target_ulong end, int flags)
2423 target_ulong addr, len;
2425 /* This function should never be called with addresses outside the
2426 guest address space. If this assert fires, it probably indicates
2427 a missing call to h2g_valid. */
2428 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2429 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2430 #endif
2431 assert(start < end);
2433 start = start & TARGET_PAGE_MASK;
2434 end = TARGET_PAGE_ALIGN(end);
2436 if (flags & PAGE_WRITE) {
2437 flags |= PAGE_WRITE_ORG;
2440 for (addr = start, len = end - start;
2441 len != 0;
2442 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2443 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2445 /* If the write protection bit is set, then we invalidate
2446 the code inside. */
2447 if (!(p->flags & PAGE_WRITE) &&
2448 (flags & PAGE_WRITE) &&
2449 p->first_tb) {
2450 tb_invalidate_phys_page(addr, 0, NULL);
2452 p->flags = flags;
2456 int page_check_range(target_ulong start, target_ulong len, int flags)
2458 PageDesc *p;
2459 target_ulong end;
2460 target_ulong addr;
2462 /* This function should never be called with addresses outside the
2463 guest address space. If this assert fires, it probably indicates
2464 a missing call to h2g_valid. */
2465 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2466 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2467 #endif
2469 if (len == 0) {
2470 return 0;
2472 if (start + len - 1 < start) {
2473 /* We've wrapped around. */
2474 return -1;
2477 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2478 start = start & TARGET_PAGE_MASK;
2480 for (addr = start, len = end - start;
2481 len != 0;
2482 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2483 p = page_find(addr >> TARGET_PAGE_BITS);
2484 if( !p )
2485 return -1;
2486 if( !(p->flags & PAGE_VALID) )
2487 return -1;
2489 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2490 return -1;
2491 if (flags & PAGE_WRITE) {
2492 if (!(p->flags & PAGE_WRITE_ORG))
2493 return -1;
2494 /* unprotect the page if it was put read-only because it
2495 contains translated code */
2496 if (!(p->flags & PAGE_WRITE)) {
2497 if (!page_unprotect(addr, 0, NULL))
2498 return -1;
2500 return 0;
2503 return 0;
2506 /* called from signal handler: invalidate the code and unprotect the
2507 page. Return TRUE if the fault was successfully handled. */
2508 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2510 unsigned int prot;
2511 PageDesc *p;
2512 target_ulong host_start, host_end, addr;
2514 /* Technically this isn't safe inside a signal handler. However we
2515 know this only ever happens in a synchronous SEGV handler, so in
2516 practice it seems to be ok. */
2517 mmap_lock();
2519 p = page_find(address >> TARGET_PAGE_BITS);
2520 if (!p) {
2521 mmap_unlock();
2522 return 0;
2525 /* if the page was really writable, then we change its
2526 protection back to writable */
2527 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2528 host_start = address & qemu_host_page_mask;
2529 host_end = host_start + qemu_host_page_size;
2531 prot = 0;
2532 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2533 p = page_find(addr >> TARGET_PAGE_BITS);
2534 p->flags |= PAGE_WRITE;
2535 prot |= p->flags;
2537 /* and since the content will be modified, we must invalidate
2538 the corresponding translated code. */
2539 tb_invalidate_phys_page(addr, pc, puc);
2540 #ifdef DEBUG_TB_CHECK
2541 tb_invalidate_check(addr);
2542 #endif
2544 mprotect((void *)g2h(host_start), qemu_host_page_size,
2545 prot & PAGE_BITS);
2547 mmap_unlock();
2548 return 1;
2550 mmap_unlock();
2551 return 0;
2554 static inline void tlb_set_dirty(CPUState *env,
2555 unsigned long addr, target_ulong vaddr)
2558 #endif /* defined(CONFIG_USER_ONLY) */
2560 #if !defined(CONFIG_USER_ONLY)
2562 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2563 typedef struct subpage_t {
2564 target_phys_addr_t base;
2565 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2566 ram_addr_t region_offset[TARGET_PAGE_SIZE];
2567 } subpage_t;
2569 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2570 ram_addr_t memory, ram_addr_t region_offset);
2571 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2572 ram_addr_t orig_memory,
2573 ram_addr_t region_offset);
2574 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2575 need_subpage) \
2576 do { \
2577 if (addr > start_addr) \
2578 start_addr2 = 0; \
2579 else { \
2580 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2581 if (start_addr2 > 0) \
2582 need_subpage = 1; \
2585 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2586 end_addr2 = TARGET_PAGE_SIZE - 1; \
2587 else { \
2588 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2589 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2590 need_subpage = 1; \
2592 } while (0)
2594 /* register physical memory.
2595 For RAM, 'size' must be a multiple of the target page size.
2596 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2597 io memory page. The address used when calling the IO function is
2598 the offset from the start of the region, plus region_offset. Both
2599 start_addr and region_offset are rounded down to a page boundary
2600 before calculating this offset. This should not be a problem unless
2601 the low bits of start_addr and region_offset differ. */
2602 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2603 ram_addr_t size,
2604 ram_addr_t phys_offset,
2605 ram_addr_t region_offset)
2607 target_phys_addr_t addr, end_addr;
2608 PhysPageDesc *p;
2609 CPUState *env;
2610 ram_addr_t orig_size = size;
2611 subpage_t *subpage;
2613 assert(size);
2614 cpu_notify_set_memory(start_addr, size, phys_offset);
2616 if (phys_offset == IO_MEM_UNASSIGNED) {
2617 region_offset = start_addr;
2619 region_offset &= TARGET_PAGE_MASK;
2620 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2621 end_addr = start_addr + (target_phys_addr_t)size;
2623 addr = start_addr;
2624 do {
2625 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2626 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2627 ram_addr_t orig_memory = p->phys_offset;
2628 target_phys_addr_t start_addr2, end_addr2;
2629 int need_subpage = 0;
2631 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2632 need_subpage);
2633 if (need_subpage) {
2634 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2635 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2636 &p->phys_offset, orig_memory,
2637 p->region_offset);
2638 } else {
2639 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2640 >> IO_MEM_SHIFT];
2642 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2643 region_offset);
2644 p->region_offset = 0;
2645 } else {
2646 p->phys_offset = phys_offset;
2647 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2648 (phys_offset & IO_MEM_ROMD))
2649 phys_offset += TARGET_PAGE_SIZE;
2651 } else {
2652 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2653 p->phys_offset = phys_offset;
2654 p->region_offset = region_offset;
2655 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2656 (phys_offset & IO_MEM_ROMD)) {
2657 phys_offset += TARGET_PAGE_SIZE;
2658 } else {
2659 target_phys_addr_t start_addr2, end_addr2;
2660 int need_subpage = 0;
2662 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2663 end_addr2, need_subpage);
2665 if (need_subpage) {
2666 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2667 &p->phys_offset, IO_MEM_UNASSIGNED,
2668 addr & TARGET_PAGE_MASK);
2669 subpage_register(subpage, start_addr2, end_addr2,
2670 phys_offset, region_offset);
2671 p->region_offset = 0;
2675 region_offset += TARGET_PAGE_SIZE;
2676 addr += TARGET_PAGE_SIZE;
2677 } while (addr != end_addr);
2679 /* since each CPU stores ram addresses in its TLB cache, we must
2680 reset the modified entries */
2681 /* XXX: slow ! */
2682 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2683 tlb_flush(env, 1);
2687 /* XXX: temporary until new memory mapping API */
2688 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2690 PhysPageDesc *p;
2692 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2693 if (!p)
2694 return IO_MEM_UNASSIGNED;
2695 return p->phys_offset;
2698 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2700 if (kvm_enabled())
2701 kvm_coalesce_mmio_region(addr, size);
2704 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2706 if (kvm_enabled())
2707 kvm_uncoalesce_mmio_region(addr, size);
2710 void qemu_flush_coalesced_mmio_buffer(void)
2712 if (kvm_enabled())
2713 kvm_flush_coalesced_mmio_buffer();
2716 #if defined(__linux__) && !defined(TARGET_S390X)
2718 #include <sys/vfs.h>
2720 #define HUGETLBFS_MAGIC 0x958458f6
2722 static long gethugepagesize(const char *path)
2724 struct statfs fs;
2725 int ret;
2727 do {
2728 ret = statfs(path, &fs);
2729 } while (ret != 0 && errno == EINTR);
2731 if (ret != 0) {
2732 perror(path);
2733 return 0;
2736 if (fs.f_type != HUGETLBFS_MAGIC)
2737 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2739 return fs.f_bsize;
2742 static void *file_ram_alloc(RAMBlock *block,
2743 ram_addr_t memory,
2744 const char *path)
2746 char *filename;
2747 void *area;
2748 int fd;
2749 #ifdef MAP_POPULATE
2750 int flags;
2751 #endif
2752 unsigned long hpagesize;
2754 hpagesize = gethugepagesize(path);
2755 if (!hpagesize) {
2756 return NULL;
2759 if (memory < hpagesize) {
2760 return NULL;
2763 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2764 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2765 return NULL;
2768 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2769 return NULL;
2772 fd = mkstemp(filename);
2773 if (fd < 0) {
2774 perror("unable to create backing store for hugepages");
2775 free(filename);
2776 return NULL;
2778 unlink(filename);
2779 free(filename);
2781 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2784 * ftruncate is not supported by hugetlbfs in older
2785 * hosts, so don't bother bailing out on errors.
2786 * If anything goes wrong with it under other filesystems,
2787 * mmap will fail.
2789 if (ftruncate(fd, memory))
2790 perror("ftruncate");
2792 #ifdef MAP_POPULATE
2793 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2794 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2795 * to sidestep this quirk.
2797 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2798 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2799 #else
2800 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2801 #endif
2802 if (area == MAP_FAILED) {
2803 perror("file_ram_alloc: can't mmap RAM pages");
2804 close(fd);
2805 return (NULL);
2807 block->fd = fd;
2808 return area;
2810 #endif
2812 static ram_addr_t find_ram_offset(ram_addr_t size)
2814 RAMBlock *block, *next_block;
2815 ram_addr_t offset = 0, mingap = ULONG_MAX;
2817 if (QLIST_EMPTY(&ram_list.blocks))
2818 return 0;
2820 QLIST_FOREACH(block, &ram_list.blocks, next) {
2821 ram_addr_t end, next = ULONG_MAX;
2823 end = block->offset + block->length;
2825 QLIST_FOREACH(next_block, &ram_list.blocks, next) {
2826 if (next_block->offset >= end) {
2827 next = MIN(next, next_block->offset);
2830 if (next - end >= size && next - end < mingap) {
2831 offset = end;
2832 mingap = next - end;
2835 return offset;
2838 static ram_addr_t last_ram_offset(void)
2840 RAMBlock *block;
2841 ram_addr_t last = 0;
2843 QLIST_FOREACH(block, &ram_list.blocks, next)
2844 last = MAX(last, block->offset + block->length);
2846 return last;
2849 ram_addr_t qemu_ram_alloc_from_ptr(DeviceState *dev, const char *name,
2850 ram_addr_t size, void *host)
2852 RAMBlock *new_block, *block;
2854 size = TARGET_PAGE_ALIGN(size);
2855 new_block = qemu_mallocz(sizeof(*new_block));
2857 if (dev && dev->parent_bus && dev->parent_bus->info->get_dev_path) {
2858 char *id = dev->parent_bus->info->get_dev_path(dev);
2859 if (id) {
2860 snprintf(new_block->idstr, sizeof(new_block->idstr), "%s/", id);
2861 qemu_free(id);
2864 pstrcat(new_block->idstr, sizeof(new_block->idstr), name);
2866 QLIST_FOREACH(block, &ram_list.blocks, next) {
2867 if (!strcmp(block->idstr, new_block->idstr)) {
2868 fprintf(stderr, "RAMBlock \"%s\" already registered, abort!\n",
2869 new_block->idstr);
2870 abort();
2874 if (host) {
2875 new_block->host = host;
2876 new_block->flags |= RAM_PREALLOC_MASK;
2877 } else {
2878 if (mem_path) {
2879 #if defined (__linux__) && !defined(TARGET_S390X)
2880 new_block->host = file_ram_alloc(new_block, size, mem_path);
2881 if (!new_block->host) {
2882 new_block->host = qemu_vmalloc(size);
2883 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2885 #else
2886 fprintf(stderr, "-mem-path option unsupported\n");
2887 exit(1);
2888 #endif
2889 } else {
2890 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2891 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2892 new_block->host = mmap((void*)0x1000000, size,
2893 PROT_EXEC|PROT_READ|PROT_WRITE,
2894 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2895 #else
2896 new_block->host = qemu_vmalloc(size);
2897 #endif
2898 qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
2902 new_block->offset = find_ram_offset(size);
2903 new_block->length = size;
2905 QLIST_INSERT_HEAD(&ram_list.blocks, new_block, next);
2907 ram_list.phys_dirty = qemu_realloc(ram_list.phys_dirty,
2908 last_ram_offset() >> TARGET_PAGE_BITS);
2909 memset(ram_list.phys_dirty + (new_block->offset >> TARGET_PAGE_BITS),
2910 0xff, size >> TARGET_PAGE_BITS);
2912 if (kvm_enabled())
2913 kvm_setup_guest_memory(new_block->host, size);
2915 return new_block->offset;
2918 ram_addr_t qemu_ram_alloc(DeviceState *dev, const char *name, ram_addr_t size)
2920 return qemu_ram_alloc_from_ptr(dev, name, size, NULL);
2923 void qemu_ram_free(ram_addr_t addr)
2925 RAMBlock *block;
2927 QLIST_FOREACH(block, &ram_list.blocks, next) {
2928 if (addr == block->offset) {
2929 QLIST_REMOVE(block, next);
2930 if (block->flags & RAM_PREALLOC_MASK) {
2932 } else if (mem_path) {
2933 #if defined (__linux__) && !defined(TARGET_S390X)
2934 if (block->fd) {
2935 munmap(block->host, block->length);
2936 close(block->fd);
2937 } else {
2938 qemu_vfree(block->host);
2940 #else
2941 abort();
2942 #endif
2943 } else {
2944 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2945 munmap(block->host, block->length);
2946 #else
2947 qemu_vfree(block->host);
2948 #endif
2950 qemu_free(block);
2951 return;
2957 #ifndef _WIN32
2958 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
2960 RAMBlock *block;
2961 ram_addr_t offset;
2962 int flags;
2963 void *area, *vaddr;
2965 QLIST_FOREACH(block, &ram_list.blocks, next) {
2966 offset = addr - block->offset;
2967 if (offset < block->length) {
2968 vaddr = block->host + offset;
2969 if (block->flags & RAM_PREALLOC_MASK) {
2971 } else {
2972 flags = MAP_FIXED;
2973 munmap(vaddr, length);
2974 if (mem_path) {
2975 #if defined(__linux__) && !defined(TARGET_S390X)
2976 if (block->fd) {
2977 #ifdef MAP_POPULATE
2978 flags |= mem_prealloc ? MAP_POPULATE | MAP_SHARED :
2979 MAP_PRIVATE;
2980 #else
2981 flags |= MAP_PRIVATE;
2982 #endif
2983 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2984 flags, block->fd, offset);
2985 } else {
2986 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
2987 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
2988 flags, -1, 0);
2990 #else
2991 abort();
2992 #endif
2993 } else {
2994 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2995 flags |= MAP_SHARED | MAP_ANONYMOUS;
2996 area = mmap(vaddr, length, PROT_EXEC|PROT_READ|PROT_WRITE,
2997 flags, -1, 0);
2998 #else
2999 flags |= MAP_PRIVATE | MAP_ANONYMOUS;
3000 area = mmap(vaddr, length, PROT_READ | PROT_WRITE,
3001 flags, -1, 0);
3002 #endif
3004 if (area != vaddr) {
3005 fprintf(stderr, "Could not remap addr: %lx@%lx\n",
3006 length, addr);
3007 exit(1);
3009 qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
3011 return;
3015 #endif /* !_WIN32 */
3017 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3018 With the exception of the softmmu code in this file, this should
3019 only be used for local memory (e.g. video ram) that the device owns,
3020 and knows it isn't going to access beyond the end of the block.
3022 It should not be used for general purpose DMA.
3023 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
3025 void *qemu_get_ram_ptr(ram_addr_t addr)
3027 RAMBlock *block;
3029 QLIST_FOREACH(block, &ram_list.blocks, next) {
3030 if (addr - block->offset < block->length) {
3031 /* Move this entry to to start of the list. */
3032 if (block != QLIST_FIRST(&ram_list.blocks)) {
3033 QLIST_REMOVE(block, next);
3034 QLIST_INSERT_HEAD(&ram_list.blocks, block, next);
3036 return block->host + (addr - block->offset);
3040 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3041 abort();
3043 return NULL;
3046 /* Return a host pointer to ram allocated with qemu_ram_alloc.
3047 * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
3049 void *qemu_safe_ram_ptr(ram_addr_t addr)
3051 RAMBlock *block;
3053 QLIST_FOREACH(block, &ram_list.blocks, next) {
3054 if (addr - block->offset < block->length) {
3055 return block->host + (addr - block->offset);
3059 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
3060 abort();
3062 return NULL;
3065 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr)
3067 RAMBlock *block;
3068 uint8_t *host = ptr;
3070 QLIST_FOREACH(block, &ram_list.blocks, next) {
3071 if (host - block->host < block->length) {
3072 *ram_addr = block->offset + (host - block->host);
3073 return 0;
3076 return -1;
3079 /* Some of the softmmu routines need to translate from a host pointer
3080 (typically a TLB entry) back to a ram offset. */
3081 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
3083 ram_addr_t ram_addr;
3085 if (qemu_ram_addr_from_host(ptr, &ram_addr)) {
3086 fprintf(stderr, "Bad ram pointer %p\n", ptr);
3087 abort();
3089 return ram_addr;
3092 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
3094 #ifdef DEBUG_UNASSIGNED
3095 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3096 #endif
3097 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3098 do_unassigned_access(addr, 0, 0, 0, 1);
3099 #endif
3100 return 0;
3103 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
3105 #ifdef DEBUG_UNASSIGNED
3106 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3107 #endif
3108 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3109 do_unassigned_access(addr, 0, 0, 0, 2);
3110 #endif
3111 return 0;
3114 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
3116 #ifdef DEBUG_UNASSIGNED
3117 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
3118 #endif
3119 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3120 do_unassigned_access(addr, 0, 0, 0, 4);
3121 #endif
3122 return 0;
3125 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
3127 #ifdef DEBUG_UNASSIGNED
3128 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3129 #endif
3130 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3131 do_unassigned_access(addr, 1, 0, 0, 1);
3132 #endif
3135 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
3137 #ifdef DEBUG_UNASSIGNED
3138 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3139 #endif
3140 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3141 do_unassigned_access(addr, 1, 0, 0, 2);
3142 #endif
3145 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
3147 #ifdef DEBUG_UNASSIGNED
3148 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
3149 #endif
3150 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
3151 do_unassigned_access(addr, 1, 0, 0, 4);
3152 #endif
3155 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
3156 unassigned_mem_readb,
3157 unassigned_mem_readw,
3158 unassigned_mem_readl,
3161 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
3162 unassigned_mem_writeb,
3163 unassigned_mem_writew,
3164 unassigned_mem_writel,
3167 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
3168 uint32_t val)
3170 int dirty_flags;
3171 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3172 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3173 #if !defined(CONFIG_USER_ONLY)
3174 tb_invalidate_phys_page_fast(ram_addr, 1);
3175 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3176 #endif
3178 stb_p(qemu_get_ram_ptr(ram_addr), val);
3179 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3180 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3181 /* we remove the notdirty callback only if the code has been
3182 flushed */
3183 if (dirty_flags == 0xff)
3184 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3187 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
3188 uint32_t val)
3190 int dirty_flags;
3191 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3192 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3193 #if !defined(CONFIG_USER_ONLY)
3194 tb_invalidate_phys_page_fast(ram_addr, 2);
3195 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3196 #endif
3198 stw_p(qemu_get_ram_ptr(ram_addr), val);
3199 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3200 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3201 /* we remove the notdirty callback only if the code has been
3202 flushed */
3203 if (dirty_flags == 0xff)
3204 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3207 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3208 uint32_t val)
3210 int dirty_flags;
3211 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3212 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3213 #if !defined(CONFIG_USER_ONLY)
3214 tb_invalidate_phys_page_fast(ram_addr, 4);
3215 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3216 #endif
3218 stl_p(qemu_get_ram_ptr(ram_addr), val);
3219 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3220 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3221 /* we remove the notdirty callback only if the code has been
3222 flushed */
3223 if (dirty_flags == 0xff)
3224 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3227 static CPUReadMemoryFunc * const error_mem_read[3] = {
3228 NULL, /* never used */
3229 NULL, /* never used */
3230 NULL, /* never used */
3233 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3234 notdirty_mem_writeb,
3235 notdirty_mem_writew,
3236 notdirty_mem_writel,
3239 /* Generate a debug exception if a watchpoint has been hit. */
3240 static void check_watchpoint(int offset, int len_mask, int flags)
3242 CPUState *env = cpu_single_env;
3243 target_ulong pc, cs_base;
3244 TranslationBlock *tb;
3245 target_ulong vaddr;
3246 CPUWatchpoint *wp;
3247 int cpu_flags;
3249 if (env->watchpoint_hit) {
3250 /* We re-entered the check after replacing the TB. Now raise
3251 * the debug interrupt so that is will trigger after the
3252 * current instruction. */
3253 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3254 return;
3256 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3257 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3258 if ((vaddr == (wp->vaddr & len_mask) ||
3259 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3260 wp->flags |= BP_WATCHPOINT_HIT;
3261 if (!env->watchpoint_hit) {
3262 env->watchpoint_hit = wp;
3263 tb = tb_find_pc(env->mem_io_pc);
3264 if (!tb) {
3265 cpu_abort(env, "check_watchpoint: could not find TB for "
3266 "pc=%p", (void *)env->mem_io_pc);
3268 cpu_restore_state(tb, env, env->mem_io_pc);
3269 tb_phys_invalidate(tb, -1);
3270 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3271 env->exception_index = EXCP_DEBUG;
3272 } else {
3273 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3274 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3276 cpu_resume_from_signal(env, NULL);
3278 } else {
3279 wp->flags &= ~BP_WATCHPOINT_HIT;
3284 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3285 so these check for a hit then pass through to the normal out-of-line
3286 phys routines. */
3287 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3289 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3290 return ldub_phys(addr);
3293 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3295 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3296 return lduw_phys(addr);
3299 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3301 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3302 return ldl_phys(addr);
3305 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3306 uint32_t val)
3308 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3309 stb_phys(addr, val);
3312 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3313 uint32_t val)
3315 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3316 stw_phys(addr, val);
3319 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3320 uint32_t val)
3322 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3323 stl_phys(addr, val);
3326 static CPUReadMemoryFunc * const watch_mem_read[3] = {
3327 watch_mem_readb,
3328 watch_mem_readw,
3329 watch_mem_readl,
3332 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3333 watch_mem_writeb,
3334 watch_mem_writew,
3335 watch_mem_writel,
3338 static inline uint32_t subpage_readlen (subpage_t *mmio,
3339 target_phys_addr_t addr,
3340 unsigned int len)
3342 unsigned int idx = SUBPAGE_IDX(addr);
3343 #if defined(DEBUG_SUBPAGE)
3344 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3345 mmio, len, addr, idx);
3346 #endif
3348 addr += mmio->region_offset[idx];
3349 idx = mmio->sub_io_index[idx];
3350 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3353 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3354 uint32_t value, unsigned int len)
3356 unsigned int idx = SUBPAGE_IDX(addr);
3357 #if defined(DEBUG_SUBPAGE)
3358 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3359 __func__, mmio, len, addr, idx, value);
3360 #endif
3362 addr += mmio->region_offset[idx];
3363 idx = mmio->sub_io_index[idx];
3364 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3367 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3369 return subpage_readlen(opaque, addr, 0);
3372 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3373 uint32_t value)
3375 subpage_writelen(opaque, addr, value, 0);
3378 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3380 return subpage_readlen(opaque, addr, 1);
3383 static void subpage_writew (void *opaque, target_phys_addr_t addr,
3384 uint32_t value)
3386 subpage_writelen(opaque, addr, value, 1);
3389 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3391 return subpage_readlen(opaque, addr, 2);
3394 static void subpage_writel (void *opaque, target_phys_addr_t addr,
3395 uint32_t value)
3397 subpage_writelen(opaque, addr, value, 2);
3400 static CPUReadMemoryFunc * const subpage_read[] = {
3401 &subpage_readb,
3402 &subpage_readw,
3403 &subpage_readl,
3406 static CPUWriteMemoryFunc * const subpage_write[] = {
3407 &subpage_writeb,
3408 &subpage_writew,
3409 &subpage_writel,
3412 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3413 ram_addr_t memory, ram_addr_t region_offset)
3415 int idx, eidx;
3417 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3418 return -1;
3419 idx = SUBPAGE_IDX(start);
3420 eidx = SUBPAGE_IDX(end);
3421 #if defined(DEBUG_SUBPAGE)
3422 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3423 mmio, start, end, idx, eidx, memory);
3424 #endif
3425 if ((memory & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
3426 memory = IO_MEM_UNASSIGNED;
3427 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3428 for (; idx <= eidx; idx++) {
3429 mmio->sub_io_index[idx] = memory;
3430 mmio->region_offset[idx] = region_offset;
3433 return 0;
3436 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3437 ram_addr_t orig_memory,
3438 ram_addr_t region_offset)
3440 subpage_t *mmio;
3441 int subpage_memory;
3443 mmio = qemu_mallocz(sizeof(subpage_t));
3445 mmio->base = base;
3446 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio,
3447 DEVICE_NATIVE_ENDIAN);
3448 #if defined(DEBUG_SUBPAGE)
3449 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3450 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3451 #endif
3452 *phys = subpage_memory | IO_MEM_SUBPAGE;
3453 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3455 return mmio;
3458 static int get_free_io_mem_idx(void)
3460 int i;
3462 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3463 if (!io_mem_used[i]) {
3464 io_mem_used[i] = 1;
3465 return i;
3467 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3468 return -1;
3472 * Usually, devices operate in little endian mode. There are devices out
3473 * there that operate in big endian too. Each device gets byte swapped
3474 * mmio if plugged onto a CPU that does the other endianness.
3476 * CPU Device swap?
3478 * little little no
3479 * little big yes
3480 * big little yes
3481 * big big no
3484 typedef struct SwapEndianContainer {
3485 CPUReadMemoryFunc *read[3];
3486 CPUWriteMemoryFunc *write[3];
3487 void *opaque;
3488 } SwapEndianContainer;
3490 static uint32_t swapendian_mem_readb (void *opaque, target_phys_addr_t addr)
3492 uint32_t val;
3493 SwapEndianContainer *c = opaque;
3494 val = c->read[0](c->opaque, addr);
3495 return val;
3498 static uint32_t swapendian_mem_readw(void *opaque, target_phys_addr_t addr)
3500 uint32_t val;
3501 SwapEndianContainer *c = opaque;
3502 val = bswap16(c->read[1](c->opaque, addr));
3503 return val;
3506 static uint32_t swapendian_mem_readl(void *opaque, target_phys_addr_t addr)
3508 uint32_t val;
3509 SwapEndianContainer *c = opaque;
3510 val = bswap32(c->read[2](c->opaque, addr));
3511 return val;
3514 static CPUReadMemoryFunc * const swapendian_readfn[3]={
3515 swapendian_mem_readb,
3516 swapendian_mem_readw,
3517 swapendian_mem_readl
3520 static void swapendian_mem_writeb(void *opaque, target_phys_addr_t addr,
3521 uint32_t val)
3523 SwapEndianContainer *c = opaque;
3524 c->write[0](c->opaque, addr, val);
3527 static void swapendian_mem_writew(void *opaque, target_phys_addr_t addr,
3528 uint32_t val)
3530 SwapEndianContainer *c = opaque;
3531 c->write[1](c->opaque, addr, bswap16(val));
3534 static void swapendian_mem_writel(void *opaque, target_phys_addr_t addr,
3535 uint32_t val)
3537 SwapEndianContainer *c = opaque;
3538 c->write[2](c->opaque, addr, bswap32(val));
3541 static CPUWriteMemoryFunc * const swapendian_writefn[3]={
3542 swapendian_mem_writeb,
3543 swapendian_mem_writew,
3544 swapendian_mem_writel
3547 static void swapendian_init(int io_index)
3549 SwapEndianContainer *c = qemu_malloc(sizeof(SwapEndianContainer));
3550 int i;
3552 /* Swap mmio for big endian targets */
3553 c->opaque = io_mem_opaque[io_index];
3554 for (i = 0; i < 3; i++) {
3555 c->read[i] = io_mem_read[io_index][i];
3556 c->write[i] = io_mem_write[io_index][i];
3558 io_mem_read[io_index][i] = swapendian_readfn[i];
3559 io_mem_write[io_index][i] = swapendian_writefn[i];
3561 io_mem_opaque[io_index] = c;
3564 static void swapendian_del(int io_index)
3566 if (io_mem_read[io_index][0] == swapendian_readfn[0]) {
3567 qemu_free(io_mem_opaque[io_index]);
3571 /* mem_read and mem_write are arrays of functions containing the
3572 function to access byte (index 0), word (index 1) and dword (index
3573 2). Functions can be omitted with a NULL function pointer.
3574 If io_index is non zero, the corresponding io zone is
3575 modified. If it is zero, a new io zone is allocated. The return
3576 value can be used with cpu_register_physical_memory(). (-1) is
3577 returned if error. */
3578 static int cpu_register_io_memory_fixed(int io_index,
3579 CPUReadMemoryFunc * const *mem_read,
3580 CPUWriteMemoryFunc * const *mem_write,
3581 void *opaque, enum device_endian endian)
3583 int i;
3585 if (io_index <= 0) {
3586 io_index = get_free_io_mem_idx();
3587 if (io_index == -1)
3588 return io_index;
3589 } else {
3590 io_index >>= IO_MEM_SHIFT;
3591 if (io_index >= IO_MEM_NB_ENTRIES)
3592 return -1;
3595 for (i = 0; i < 3; ++i) {
3596 io_mem_read[io_index][i]
3597 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3599 for (i = 0; i < 3; ++i) {
3600 io_mem_write[io_index][i]
3601 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3603 io_mem_opaque[io_index] = opaque;
3605 switch (endian) {
3606 case DEVICE_BIG_ENDIAN:
3607 #ifndef TARGET_WORDS_BIGENDIAN
3608 swapendian_init(io_index);
3609 #endif
3610 break;
3611 case DEVICE_LITTLE_ENDIAN:
3612 #ifdef TARGET_WORDS_BIGENDIAN
3613 swapendian_init(io_index);
3614 #endif
3615 break;
3616 case DEVICE_NATIVE_ENDIAN:
3617 default:
3618 break;
3621 return (io_index << IO_MEM_SHIFT);
3624 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3625 CPUWriteMemoryFunc * const *mem_write,
3626 void *opaque, enum device_endian endian)
3628 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque, endian);
3631 void cpu_unregister_io_memory(int io_table_address)
3633 int i;
3634 int io_index = io_table_address >> IO_MEM_SHIFT;
3636 swapendian_del(io_index);
3638 for (i=0;i < 3; i++) {
3639 io_mem_read[io_index][i] = unassigned_mem_read[i];
3640 io_mem_write[io_index][i] = unassigned_mem_write[i];
3642 io_mem_opaque[io_index] = NULL;
3643 io_mem_used[io_index] = 0;
3646 static void io_mem_init(void)
3648 int i;
3650 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read,
3651 unassigned_mem_write, NULL,
3652 DEVICE_NATIVE_ENDIAN);
3653 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read,
3654 unassigned_mem_write, NULL,
3655 DEVICE_NATIVE_ENDIAN);
3656 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read,
3657 notdirty_mem_write, NULL,
3658 DEVICE_NATIVE_ENDIAN);
3659 for (i=0; i<5; i++)
3660 io_mem_used[i] = 1;
3662 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3663 watch_mem_write, NULL,
3664 DEVICE_NATIVE_ENDIAN);
3667 #endif /* !defined(CONFIG_USER_ONLY) */
3669 /* physical memory access (slow version, mainly for debug) */
3670 #if defined(CONFIG_USER_ONLY)
3671 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3672 uint8_t *buf, int len, int is_write)
3674 int l, flags;
3675 target_ulong page;
3676 void * p;
3678 while (len > 0) {
3679 page = addr & TARGET_PAGE_MASK;
3680 l = (page + TARGET_PAGE_SIZE) - addr;
3681 if (l > len)
3682 l = len;
3683 flags = page_get_flags(page);
3684 if (!(flags & PAGE_VALID))
3685 return -1;
3686 if (is_write) {
3687 if (!(flags & PAGE_WRITE))
3688 return -1;
3689 /* XXX: this code should not depend on lock_user */
3690 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3691 return -1;
3692 memcpy(p, buf, l);
3693 unlock_user(p, addr, l);
3694 } else {
3695 if (!(flags & PAGE_READ))
3696 return -1;
3697 /* XXX: this code should not depend on lock_user */
3698 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3699 return -1;
3700 memcpy(buf, p, l);
3701 unlock_user(p, addr, 0);
3703 len -= l;
3704 buf += l;
3705 addr += l;
3707 return 0;
3710 #else
3711 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3712 int len, int is_write)
3714 int l, io_index;
3715 uint8_t *ptr;
3716 uint32_t val;
3717 target_phys_addr_t page;
3718 unsigned long pd;
3719 PhysPageDesc *p;
3721 while (len > 0) {
3722 page = addr & TARGET_PAGE_MASK;
3723 l = (page + TARGET_PAGE_SIZE) - addr;
3724 if (l > len)
3725 l = len;
3726 p = phys_page_find(page >> TARGET_PAGE_BITS);
3727 if (!p) {
3728 pd = IO_MEM_UNASSIGNED;
3729 } else {
3730 pd = p->phys_offset;
3733 if (is_write) {
3734 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3735 target_phys_addr_t addr1 = addr;
3736 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3737 if (p)
3738 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3739 /* XXX: could force cpu_single_env to NULL to avoid
3740 potential bugs */
3741 if (l >= 4 && ((addr1 & 3) == 0)) {
3742 /* 32 bit write access */
3743 val = ldl_p(buf);
3744 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3745 l = 4;
3746 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3747 /* 16 bit write access */
3748 val = lduw_p(buf);
3749 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3750 l = 2;
3751 } else {
3752 /* 8 bit write access */
3753 val = ldub_p(buf);
3754 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3755 l = 1;
3757 } else {
3758 unsigned long addr1;
3759 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3760 /* RAM case */
3761 ptr = qemu_get_ram_ptr(addr1);
3762 memcpy(ptr, buf, l);
3763 if (!cpu_physical_memory_is_dirty(addr1)) {
3764 /* invalidate code */
3765 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3766 /* set dirty bit */
3767 cpu_physical_memory_set_dirty_flags(
3768 addr1, (0xff & ~CODE_DIRTY_FLAG));
3771 } else {
3772 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3773 !(pd & IO_MEM_ROMD)) {
3774 target_phys_addr_t addr1 = addr;
3775 /* I/O case */
3776 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3777 if (p)
3778 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3779 if (l >= 4 && ((addr1 & 3) == 0)) {
3780 /* 32 bit read access */
3781 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3782 stl_p(buf, val);
3783 l = 4;
3784 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3785 /* 16 bit read access */
3786 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3787 stw_p(buf, val);
3788 l = 2;
3789 } else {
3790 /* 8 bit read access */
3791 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3792 stb_p(buf, val);
3793 l = 1;
3795 } else {
3796 /* RAM case */
3797 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3798 (addr & ~TARGET_PAGE_MASK);
3799 memcpy(buf, ptr, l);
3802 len -= l;
3803 buf += l;
3804 addr += l;
3808 /* used for ROM loading : can write in RAM and ROM */
3809 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3810 const uint8_t *buf, int len)
3812 int l;
3813 uint8_t *ptr;
3814 target_phys_addr_t page;
3815 unsigned long pd;
3816 PhysPageDesc *p;
3818 while (len > 0) {
3819 page = addr & TARGET_PAGE_MASK;
3820 l = (page + TARGET_PAGE_SIZE) - addr;
3821 if (l > len)
3822 l = len;
3823 p = phys_page_find(page >> TARGET_PAGE_BITS);
3824 if (!p) {
3825 pd = IO_MEM_UNASSIGNED;
3826 } else {
3827 pd = p->phys_offset;
3830 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3831 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3832 !(pd & IO_MEM_ROMD)) {
3833 /* do nothing */
3834 } else {
3835 unsigned long addr1;
3836 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3837 /* ROM/RAM case */
3838 ptr = qemu_get_ram_ptr(addr1);
3839 memcpy(ptr, buf, l);
3841 len -= l;
3842 buf += l;
3843 addr += l;
3847 typedef struct {
3848 void *buffer;
3849 target_phys_addr_t addr;
3850 target_phys_addr_t len;
3851 } BounceBuffer;
3853 static BounceBuffer bounce;
3855 typedef struct MapClient {
3856 void *opaque;
3857 void (*callback)(void *opaque);
3858 QLIST_ENTRY(MapClient) link;
3859 } MapClient;
3861 static QLIST_HEAD(map_client_list, MapClient) map_client_list
3862 = QLIST_HEAD_INITIALIZER(map_client_list);
3864 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3866 MapClient *client = qemu_malloc(sizeof(*client));
3868 client->opaque = opaque;
3869 client->callback = callback;
3870 QLIST_INSERT_HEAD(&map_client_list, client, link);
3871 return client;
3874 void cpu_unregister_map_client(void *_client)
3876 MapClient *client = (MapClient *)_client;
3878 QLIST_REMOVE(client, link);
3879 qemu_free(client);
3882 static void cpu_notify_map_clients(void)
3884 MapClient *client;
3886 while (!QLIST_EMPTY(&map_client_list)) {
3887 client = QLIST_FIRST(&map_client_list);
3888 client->callback(client->opaque);
3889 cpu_unregister_map_client(client);
3893 /* Map a physical memory region into a host virtual address.
3894 * May map a subset of the requested range, given by and returned in *plen.
3895 * May return NULL if resources needed to perform the mapping are exhausted.
3896 * Use only for reads OR writes - not for read-modify-write operations.
3897 * Use cpu_register_map_client() to know when retrying the map operation is
3898 * likely to succeed.
3900 void *cpu_physical_memory_map(target_phys_addr_t addr,
3901 target_phys_addr_t *plen,
3902 int is_write)
3904 target_phys_addr_t len = *plen;
3905 target_phys_addr_t done = 0;
3906 int l;
3907 uint8_t *ret = NULL;
3908 uint8_t *ptr;
3909 target_phys_addr_t page;
3910 unsigned long pd;
3911 PhysPageDesc *p;
3912 unsigned long addr1;
3914 while (len > 0) {
3915 page = addr & TARGET_PAGE_MASK;
3916 l = (page + TARGET_PAGE_SIZE) - addr;
3917 if (l > len)
3918 l = len;
3919 p = phys_page_find(page >> TARGET_PAGE_BITS);
3920 if (!p) {
3921 pd = IO_MEM_UNASSIGNED;
3922 } else {
3923 pd = p->phys_offset;
3926 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3927 if (done || bounce.buffer) {
3928 break;
3930 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3931 bounce.addr = addr;
3932 bounce.len = l;
3933 if (!is_write) {
3934 cpu_physical_memory_read(addr, bounce.buffer, l);
3936 ptr = bounce.buffer;
3937 } else {
3938 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3939 ptr = qemu_get_ram_ptr(addr1);
3941 if (!done) {
3942 ret = ptr;
3943 } else if (ret + done != ptr) {
3944 break;
3947 len -= l;
3948 addr += l;
3949 done += l;
3951 *plen = done;
3952 return ret;
3955 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3956 * Will also mark the memory as dirty if is_write == 1. access_len gives
3957 * the amount of memory that was actually read or written by the caller.
3959 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3960 int is_write, target_phys_addr_t access_len)
3962 if (buffer != bounce.buffer) {
3963 if (is_write) {
3964 ram_addr_t addr1 = qemu_ram_addr_from_host_nofail(buffer);
3965 while (access_len) {
3966 unsigned l;
3967 l = TARGET_PAGE_SIZE;
3968 if (l > access_len)
3969 l = access_len;
3970 if (!cpu_physical_memory_is_dirty(addr1)) {
3971 /* invalidate code */
3972 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3973 /* set dirty bit */
3974 cpu_physical_memory_set_dirty_flags(
3975 addr1, (0xff & ~CODE_DIRTY_FLAG));
3977 addr1 += l;
3978 access_len -= l;
3981 return;
3983 if (is_write) {
3984 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3986 qemu_vfree(bounce.buffer);
3987 bounce.buffer = NULL;
3988 cpu_notify_map_clients();
3991 /* warning: addr must be aligned */
3992 uint32_t ldl_phys(target_phys_addr_t addr)
3994 int io_index;
3995 uint8_t *ptr;
3996 uint32_t val;
3997 unsigned long pd;
3998 PhysPageDesc *p;
4000 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4001 if (!p) {
4002 pd = IO_MEM_UNASSIGNED;
4003 } else {
4004 pd = p->phys_offset;
4007 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4008 !(pd & IO_MEM_ROMD)) {
4009 /* I/O case */
4010 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4011 if (p)
4012 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4013 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4014 } else {
4015 /* RAM case */
4016 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4017 (addr & ~TARGET_PAGE_MASK);
4018 val = ldl_p(ptr);
4020 return val;
4023 /* warning: addr must be aligned */
4024 uint64_t ldq_phys(target_phys_addr_t addr)
4026 int io_index;
4027 uint8_t *ptr;
4028 uint64_t val;
4029 unsigned long pd;
4030 PhysPageDesc *p;
4032 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4033 if (!p) {
4034 pd = IO_MEM_UNASSIGNED;
4035 } else {
4036 pd = p->phys_offset;
4039 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4040 !(pd & IO_MEM_ROMD)) {
4041 /* I/O case */
4042 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4043 if (p)
4044 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4045 #ifdef TARGET_WORDS_BIGENDIAN
4046 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
4047 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
4048 #else
4049 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
4050 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
4051 #endif
4052 } else {
4053 /* RAM case */
4054 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4055 (addr & ~TARGET_PAGE_MASK);
4056 val = ldq_p(ptr);
4058 return val;
4061 /* XXX: optimize */
4062 uint32_t ldub_phys(target_phys_addr_t addr)
4064 uint8_t val;
4065 cpu_physical_memory_read(addr, &val, 1);
4066 return val;
4069 /* warning: addr must be aligned */
4070 uint32_t lduw_phys(target_phys_addr_t addr)
4072 int io_index;
4073 uint8_t *ptr;
4074 uint64_t val;
4075 unsigned long pd;
4076 PhysPageDesc *p;
4078 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4079 if (!p) {
4080 pd = IO_MEM_UNASSIGNED;
4081 } else {
4082 pd = p->phys_offset;
4085 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
4086 !(pd & IO_MEM_ROMD)) {
4087 /* I/O case */
4088 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4089 if (p)
4090 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4091 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
4092 } else {
4093 /* RAM case */
4094 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4095 (addr & ~TARGET_PAGE_MASK);
4096 val = lduw_p(ptr);
4098 return val;
4101 /* warning: addr must be aligned. The ram page is not masked as dirty
4102 and the code inside is not invalidated. It is useful if the dirty
4103 bits are used to track modified PTEs */
4104 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
4106 int io_index;
4107 uint8_t *ptr;
4108 unsigned long pd;
4109 PhysPageDesc *p;
4111 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4112 if (!p) {
4113 pd = IO_MEM_UNASSIGNED;
4114 } else {
4115 pd = p->phys_offset;
4118 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4119 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4120 if (p)
4121 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4122 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4123 } else {
4124 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4125 ptr = qemu_get_ram_ptr(addr1);
4126 stl_p(ptr, val);
4128 if (unlikely(in_migration)) {
4129 if (!cpu_physical_memory_is_dirty(addr1)) {
4130 /* invalidate code */
4131 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4132 /* set dirty bit */
4133 cpu_physical_memory_set_dirty_flags(
4134 addr1, (0xff & ~CODE_DIRTY_FLAG));
4140 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
4142 int io_index;
4143 uint8_t *ptr;
4144 unsigned long pd;
4145 PhysPageDesc *p;
4147 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4148 if (!p) {
4149 pd = IO_MEM_UNASSIGNED;
4150 } else {
4151 pd = p->phys_offset;
4154 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4155 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4156 if (p)
4157 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4158 #ifdef TARGET_WORDS_BIGENDIAN
4159 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
4160 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
4161 #else
4162 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4163 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
4164 #endif
4165 } else {
4166 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
4167 (addr & ~TARGET_PAGE_MASK);
4168 stq_p(ptr, val);
4172 /* warning: addr must be aligned */
4173 void stl_phys(target_phys_addr_t addr, uint32_t val)
4175 int io_index;
4176 uint8_t *ptr;
4177 unsigned long pd;
4178 PhysPageDesc *p;
4180 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4181 if (!p) {
4182 pd = IO_MEM_UNASSIGNED;
4183 } else {
4184 pd = p->phys_offset;
4187 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4188 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4189 if (p)
4190 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4191 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
4192 } else {
4193 unsigned long addr1;
4194 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4195 /* RAM case */
4196 ptr = qemu_get_ram_ptr(addr1);
4197 stl_p(ptr, val);
4198 if (!cpu_physical_memory_is_dirty(addr1)) {
4199 /* invalidate code */
4200 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
4201 /* set dirty bit */
4202 cpu_physical_memory_set_dirty_flags(addr1,
4203 (0xff & ~CODE_DIRTY_FLAG));
4208 /* XXX: optimize */
4209 void stb_phys(target_phys_addr_t addr, uint32_t val)
4211 uint8_t v = val;
4212 cpu_physical_memory_write(addr, &v, 1);
4215 /* warning: addr must be aligned */
4216 void stw_phys(target_phys_addr_t addr, uint32_t val)
4218 int io_index;
4219 uint8_t *ptr;
4220 unsigned long pd;
4221 PhysPageDesc *p;
4223 p = phys_page_find(addr >> TARGET_PAGE_BITS);
4224 if (!p) {
4225 pd = IO_MEM_UNASSIGNED;
4226 } else {
4227 pd = p->phys_offset;
4230 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
4231 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
4232 if (p)
4233 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
4234 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
4235 } else {
4236 unsigned long addr1;
4237 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
4238 /* RAM case */
4239 ptr = qemu_get_ram_ptr(addr1);
4240 stw_p(ptr, val);
4241 if (!cpu_physical_memory_is_dirty(addr1)) {
4242 /* invalidate code */
4243 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
4244 /* set dirty bit */
4245 cpu_physical_memory_set_dirty_flags(addr1,
4246 (0xff & ~CODE_DIRTY_FLAG));
4251 /* XXX: optimize */
4252 void stq_phys(target_phys_addr_t addr, uint64_t val)
4254 val = tswap64(val);
4255 cpu_physical_memory_write(addr, &val, 8);
4258 /* virtual memory access for debug (includes writing to ROM) */
4259 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
4260 uint8_t *buf, int len, int is_write)
4262 int l;
4263 target_phys_addr_t phys_addr;
4264 target_ulong page;
4266 while (len > 0) {
4267 page = addr & TARGET_PAGE_MASK;
4268 phys_addr = cpu_get_phys_page_debug(env, page);
4269 /* if no physical page mapped, return an error */
4270 if (phys_addr == -1)
4271 return -1;
4272 l = (page + TARGET_PAGE_SIZE) - addr;
4273 if (l > len)
4274 l = len;
4275 phys_addr += (addr & ~TARGET_PAGE_MASK);
4276 if (is_write)
4277 cpu_physical_memory_write_rom(phys_addr, buf, l);
4278 else
4279 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
4280 len -= l;
4281 buf += l;
4282 addr += l;
4284 return 0;
4286 #endif
4288 /* in deterministic execution mode, instructions doing device I/Os
4289 must be at the end of the TB */
4290 void cpu_io_recompile(CPUState *env, void *retaddr)
4292 TranslationBlock *tb;
4293 uint32_t n, cflags;
4294 target_ulong pc, cs_base;
4295 uint64_t flags;
4297 tb = tb_find_pc((unsigned long)retaddr);
4298 if (!tb) {
4299 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
4300 retaddr);
4302 n = env->icount_decr.u16.low + tb->icount;
4303 cpu_restore_state(tb, env, (unsigned long)retaddr);
4304 /* Calculate how many instructions had been executed before the fault
4305 occurred. */
4306 n = n - env->icount_decr.u16.low;
4307 /* Generate a new TB ending on the I/O insn. */
4308 n++;
4309 /* On MIPS and SH, delay slot instructions can only be restarted if
4310 they were already the first instruction in the TB. If this is not
4311 the first instruction in a TB then re-execute the preceding
4312 branch. */
4313 #if defined(TARGET_MIPS)
4314 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
4315 env->active_tc.PC -= 4;
4316 env->icount_decr.u16.low++;
4317 env->hflags &= ~MIPS_HFLAG_BMASK;
4319 #elif defined(TARGET_SH4)
4320 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
4321 && n > 1) {
4322 env->pc -= 2;
4323 env->icount_decr.u16.low++;
4324 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
4326 #endif
4327 /* This should never happen. */
4328 if (n > CF_COUNT_MASK)
4329 cpu_abort(env, "TB too big during recompile");
4331 cflags = n | CF_LAST_IO;
4332 pc = tb->pc;
4333 cs_base = tb->cs_base;
4334 flags = tb->flags;
4335 tb_phys_invalidate(tb, -1);
4336 /* FIXME: In theory this could raise an exception. In practice
4337 we have already translated the block once so it's probably ok. */
4338 tb_gen_code(env, pc, cs_base, flags, cflags);
4339 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4340 the first in the TB) then we end up generating a whole new TB and
4341 repeating the fault, which is horribly inefficient.
4342 Better would be to execute just this insn uncached, or generate a
4343 second new TB. */
4344 cpu_resume_from_signal(env, NULL);
4347 #if !defined(CONFIG_USER_ONLY)
4349 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
4351 int i, target_code_size, max_target_code_size;
4352 int direct_jmp_count, direct_jmp2_count, cross_page;
4353 TranslationBlock *tb;
4355 target_code_size = 0;
4356 max_target_code_size = 0;
4357 cross_page = 0;
4358 direct_jmp_count = 0;
4359 direct_jmp2_count = 0;
4360 for(i = 0; i < nb_tbs; i++) {
4361 tb = &tbs[i];
4362 target_code_size += tb->size;
4363 if (tb->size > max_target_code_size)
4364 max_target_code_size = tb->size;
4365 if (tb->page_addr[1] != -1)
4366 cross_page++;
4367 if (tb->tb_next_offset[0] != 0xffff) {
4368 direct_jmp_count++;
4369 if (tb->tb_next_offset[1] != 0xffff) {
4370 direct_jmp2_count++;
4374 /* XXX: avoid using doubles ? */
4375 cpu_fprintf(f, "Translation buffer state:\n");
4376 cpu_fprintf(f, "gen code size %td/%ld\n",
4377 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4378 cpu_fprintf(f, "TB count %d/%d\n",
4379 nb_tbs, code_gen_max_blocks);
4380 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4381 nb_tbs ? target_code_size / nb_tbs : 0,
4382 max_target_code_size);
4383 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
4384 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4385 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4386 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4387 cross_page,
4388 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4389 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4390 direct_jmp_count,
4391 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4392 direct_jmp2_count,
4393 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4394 cpu_fprintf(f, "\nStatistics:\n");
4395 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4396 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4397 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4398 tcg_dump_info(f, cpu_fprintf);
4401 #define MMUSUFFIX _cmmu
4402 #define GETPC() NULL
4403 #define env cpu_single_env
4404 #define SOFTMMU_CODE_ACCESS
4406 #define SHIFT 0
4407 #include "softmmu_template.h"
4409 #define SHIFT 1
4410 #include "softmmu_template.h"
4412 #define SHIFT 2
4413 #include "softmmu_template.h"
4415 #define SHIFT 3
4416 #include "softmmu_template.h"
4418 #undef env
4420 #endif