Fix overflow in i440fx_init()
[qemu/lumag.git] / exec.c
blobbb3dcadcded839ce3bf7a4507ab5b3fe8a32202c
1 /*
2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <stdarg.h>
29 #include <string.h>
30 #include <errno.h>
31 #include <unistd.h>
32 #include <inttypes.h>
34 #include "cpu.h"
35 #include "exec-all.h"
36 #include "qemu-common.h"
37 #include "tcg.h"
38 #include "hw/hw.h"
39 #include "osdep.h"
40 #include "kvm.h"
41 #include "qemu-timer.h"
42 #if defined(CONFIG_USER_ONLY)
43 #include <qemu.h>
44 #include <signal.h>
45 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
46 #include <sys/param.h>
47 #if __FreeBSD_version >= 700104
48 #define HAVE_KINFO_GETVMMAP
49 #define sigqueue sigqueue_freebsd /* avoid redefinition */
50 #include <sys/time.h>
51 #include <sys/proc.h>
52 #include <machine/profile.h>
53 #define _KERNEL
54 #include <sys/user.h>
55 #undef _KERNEL
56 #undef sigqueue
57 #include <libutil.h>
58 #endif
59 #endif
60 #endif
62 //#define DEBUG_TB_INVALIDATE
63 //#define DEBUG_FLUSH
64 //#define DEBUG_TLB
65 //#define DEBUG_UNASSIGNED
67 /* make various TB consistency checks */
68 //#define DEBUG_TB_CHECK
69 //#define DEBUG_TLB_CHECK
71 //#define DEBUG_IOPORT
72 //#define DEBUG_SUBPAGE
74 #if !defined(CONFIG_USER_ONLY)
75 /* TB consistency checks only implemented for usermode emulation. */
76 #undef DEBUG_TB_CHECK
77 #endif
79 #define SMC_BITMAP_USE_THRESHOLD 10
81 static TranslationBlock *tbs;
82 int code_gen_max_blocks;
83 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
84 static int nb_tbs;
85 /* any access to the tbs or the page table must use this lock */
86 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
88 #if defined(__arm__) || defined(__sparc_v9__)
89 /* The prologue must be reachable with a direct jump. ARM and Sparc64
90 have limited branch ranges (possibly also PPC) so place it in a
91 section close to code segment. */
92 #define code_gen_section \
93 __attribute__((__section__(".gen_code"))) \
94 __attribute__((aligned (32)))
95 #elif defined(_WIN32)
96 /* Maximum alignment for Win32 is 16. */
97 #define code_gen_section \
98 __attribute__((aligned (16)))
99 #else
100 #define code_gen_section \
101 __attribute__((aligned (32)))
102 #endif
104 uint8_t code_gen_prologue[1024] code_gen_section;
105 static uint8_t *code_gen_buffer;
106 static unsigned long code_gen_buffer_size;
107 /* threshold to flush the translated code buffer */
108 static unsigned long code_gen_buffer_max_size;
109 uint8_t *code_gen_ptr;
111 #if !defined(CONFIG_USER_ONLY)
112 int phys_ram_fd;
113 uint8_t *phys_ram_dirty;
114 static int in_migration;
116 typedef struct RAMBlock {
117 uint8_t *host;
118 ram_addr_t offset;
119 ram_addr_t length;
120 struct RAMBlock *next;
121 } RAMBlock;
123 static RAMBlock *ram_blocks;
124 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
125 then we can no longer assume contiguous ram offsets, and external uses
126 of this variable will break. */
127 ram_addr_t last_ram_offset;
128 #endif
130 CPUState *first_cpu;
131 /* current CPU in the current thread. It is only valid inside
132 cpu_exec() */
133 CPUState *cpu_single_env;
134 /* 0 = Do not count executed instructions.
135 1 = Precise instruction counting.
136 2 = Adaptive rate instruction counting. */
137 int use_icount = 0;
138 /* Current instruction counter. While executing translated code this may
139 include some instructions that have not yet been executed. */
140 int64_t qemu_icount;
142 typedef struct PageDesc {
143 /* list of TBs intersecting this ram page */
144 TranslationBlock *first_tb;
145 /* in order to optimize self modifying code, we count the number
146 of lookups we do to a given page to use a bitmap */
147 unsigned int code_write_count;
148 uint8_t *code_bitmap;
149 #if defined(CONFIG_USER_ONLY)
150 unsigned long flags;
151 #endif
152 } PageDesc;
154 /* In system mode we want L1_MAP to be based on ram offsets,
155 while in user mode we want it to be based on virtual addresses. */
156 #if !defined(CONFIG_USER_ONLY)
157 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
158 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
159 #else
160 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
161 #endif
162 #else
163 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
164 #endif
166 /* Size of the L2 (and L3, etc) page tables. */
167 #define L2_BITS 10
168 #define L2_SIZE (1 << L2_BITS)
170 /* The bits remaining after N lower levels of page tables. */
171 #define P_L1_BITS_REM \
172 ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
173 #define V_L1_BITS_REM \
174 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
176 /* Size of the L1 page table. Avoid silly small sizes. */
177 #if P_L1_BITS_REM < 4
178 #define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
179 #else
180 #define P_L1_BITS P_L1_BITS_REM
181 #endif
183 #if V_L1_BITS_REM < 4
184 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
185 #else
186 #define V_L1_BITS V_L1_BITS_REM
187 #endif
189 #define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
190 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
192 #define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
193 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
195 unsigned long qemu_real_host_page_size;
196 unsigned long qemu_host_page_bits;
197 unsigned long qemu_host_page_size;
198 unsigned long qemu_host_page_mask;
200 /* This is a multi-level map on the virtual address space.
201 The bottom level has pointers to PageDesc. */
202 static void *l1_map[V_L1_SIZE];
204 #if !defined(CONFIG_USER_ONLY)
205 typedef struct PhysPageDesc {
206 /* offset in host memory of the page + io_index in the low bits */
207 ram_addr_t phys_offset;
208 ram_addr_t region_offset;
209 } PhysPageDesc;
211 /* This is a multi-level map on the physical address space.
212 The bottom level has pointers to PhysPageDesc. */
213 static void *l1_phys_map[P_L1_SIZE];
215 static void io_mem_init(void);
217 /* io memory support */
218 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
219 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
220 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
221 static char io_mem_used[IO_MEM_NB_ENTRIES];
222 static int io_mem_watch;
223 #endif
225 /* log support */
226 #ifdef WIN32
227 static const char *logfilename = "qemu.log";
228 #else
229 static const char *logfilename = "/tmp/qemu.log";
230 #endif
231 FILE *logfile;
232 int loglevel;
233 static int log_append = 0;
235 /* statistics */
236 #if !defined(CONFIG_USER_ONLY)
237 static int tlb_flush_count;
238 #endif
239 static int tb_flush_count;
240 static int tb_phys_invalidate_count;
242 #ifdef _WIN32
243 static void map_exec(void *addr, long size)
245 DWORD old_protect;
246 VirtualProtect(addr, size,
247 PAGE_EXECUTE_READWRITE, &old_protect);
250 #else
251 static void map_exec(void *addr, long size)
253 unsigned long start, end, page_size;
255 page_size = getpagesize();
256 start = (unsigned long)addr;
257 start &= ~(page_size - 1);
259 end = (unsigned long)addr + size;
260 end += page_size - 1;
261 end &= ~(page_size - 1);
263 mprotect((void *)start, end - start,
264 PROT_READ | PROT_WRITE | PROT_EXEC);
266 #endif
268 static void page_init(void)
270 /* NOTE: we can always suppose that qemu_host_page_size >=
271 TARGET_PAGE_SIZE */
272 #ifdef _WIN32
274 SYSTEM_INFO system_info;
276 GetSystemInfo(&system_info);
277 qemu_real_host_page_size = system_info.dwPageSize;
279 #else
280 qemu_real_host_page_size = getpagesize();
281 #endif
282 if (qemu_host_page_size == 0)
283 qemu_host_page_size = qemu_real_host_page_size;
284 if (qemu_host_page_size < TARGET_PAGE_SIZE)
285 qemu_host_page_size = TARGET_PAGE_SIZE;
286 qemu_host_page_bits = 0;
287 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
288 qemu_host_page_bits++;
289 qemu_host_page_mask = ~(qemu_host_page_size - 1);
291 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
293 #ifdef HAVE_KINFO_GETVMMAP
294 struct kinfo_vmentry *freep;
295 int i, cnt;
297 freep = kinfo_getvmmap(getpid(), &cnt);
298 if (freep) {
299 mmap_lock();
300 for (i = 0; i < cnt; i++) {
301 unsigned long startaddr, endaddr;
303 startaddr = freep[i].kve_start;
304 endaddr = freep[i].kve_end;
305 if (h2g_valid(startaddr)) {
306 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
308 if (h2g_valid(endaddr)) {
309 endaddr = h2g(endaddr);
310 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
311 } else {
312 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
313 endaddr = ~0ul;
314 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
315 #endif
319 free(freep);
320 mmap_unlock();
322 #else
323 FILE *f;
325 last_brk = (unsigned long)sbrk(0);
327 f = fopen("/compat/linux/proc/self/maps", "r");
328 if (f) {
329 mmap_lock();
331 do {
332 unsigned long startaddr, endaddr;
333 int n;
335 n = fscanf (f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
337 if (n == 2 && h2g_valid(startaddr)) {
338 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
340 if (h2g_valid(endaddr)) {
341 endaddr = h2g(endaddr);
342 } else {
343 endaddr = ~0ul;
345 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
347 } while (!feof(f));
349 fclose(f);
350 mmap_unlock();
352 #endif
354 #endif
357 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
359 PageDesc *pd;
360 void **lp;
361 int i;
363 #if defined(CONFIG_USER_ONLY)
364 /* We can't use qemu_malloc because it may recurse into a locked mutex. */
365 # define ALLOC(P, SIZE) \
366 do { \
367 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
368 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
369 } while (0)
370 #else
371 # define ALLOC(P, SIZE) \
372 do { P = qemu_mallocz(SIZE); } while (0)
373 #endif
375 /* Level 1. Always allocated. */
376 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
378 /* Level 2..N-1. */
379 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
380 void **p = *lp;
382 if (p == NULL) {
383 if (!alloc) {
384 return NULL;
386 ALLOC(p, sizeof(void *) * L2_SIZE);
387 *lp = p;
390 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
393 pd = *lp;
394 if (pd == NULL) {
395 if (!alloc) {
396 return NULL;
398 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
399 *lp = pd;
402 #undef ALLOC
404 return pd + (index & (L2_SIZE - 1));
407 static inline PageDesc *page_find(tb_page_addr_t index)
409 return page_find_alloc(index, 0);
412 #if !defined(CONFIG_USER_ONLY)
413 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
415 PhysPageDesc *pd;
416 void **lp;
417 int i;
419 /* Level 1. Always allocated. */
420 lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
422 /* Level 2..N-1. */
423 for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
424 void **p = *lp;
425 if (p == NULL) {
426 if (!alloc) {
427 return NULL;
429 *lp = p = qemu_mallocz(sizeof(void *) * L2_SIZE);
431 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
434 pd = *lp;
435 if (pd == NULL) {
436 int i;
438 if (!alloc) {
439 return NULL;
442 *lp = pd = qemu_malloc(sizeof(PhysPageDesc) * L2_SIZE);
444 for (i = 0; i < L2_SIZE; i++) {
445 pd[i].phys_offset = IO_MEM_UNASSIGNED;
446 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
450 return pd + (index & (L2_SIZE - 1));
453 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
455 return phys_page_find_alloc(index, 0);
458 static void tlb_protect_code(ram_addr_t ram_addr);
459 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
460 target_ulong vaddr);
461 #define mmap_lock() do { } while(0)
462 #define mmap_unlock() do { } while(0)
463 #endif
465 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
467 #if defined(CONFIG_USER_ONLY)
468 /* Currently it is not recommended to allocate big chunks of data in
469 user mode. It will change when a dedicated libc will be used */
470 #define USE_STATIC_CODE_GEN_BUFFER
471 #endif
473 #ifdef USE_STATIC_CODE_GEN_BUFFER
474 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
475 __attribute__((aligned (CODE_GEN_ALIGN)));
476 #endif
478 static void code_gen_alloc(unsigned long tb_size)
480 #ifdef USE_STATIC_CODE_GEN_BUFFER
481 code_gen_buffer = static_code_gen_buffer;
482 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
483 map_exec(code_gen_buffer, code_gen_buffer_size);
484 #else
485 code_gen_buffer_size = tb_size;
486 if (code_gen_buffer_size == 0) {
487 #if defined(CONFIG_USER_ONLY)
488 /* in user mode, phys_ram_size is not meaningful */
489 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
490 #else
491 /* XXX: needs adjustments */
492 code_gen_buffer_size = (unsigned long)(ram_size / 4);
493 #endif
495 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
496 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
497 /* The code gen buffer location may have constraints depending on
498 the host cpu and OS */
499 #if defined(__linux__)
501 int flags;
502 void *start = NULL;
504 flags = MAP_PRIVATE | MAP_ANONYMOUS;
505 #if defined(__x86_64__)
506 flags |= MAP_32BIT;
507 /* Cannot map more than that */
508 if (code_gen_buffer_size > (800 * 1024 * 1024))
509 code_gen_buffer_size = (800 * 1024 * 1024);
510 #elif defined(__sparc_v9__)
511 // Map the buffer below 2G, so we can use direct calls and branches
512 flags |= MAP_FIXED;
513 start = (void *) 0x60000000UL;
514 if (code_gen_buffer_size > (512 * 1024 * 1024))
515 code_gen_buffer_size = (512 * 1024 * 1024);
516 #elif defined(__arm__)
517 /* Map the buffer below 32M, so we can use direct calls and branches */
518 flags |= MAP_FIXED;
519 start = (void *) 0x01000000UL;
520 if (code_gen_buffer_size > 16 * 1024 * 1024)
521 code_gen_buffer_size = 16 * 1024 * 1024;
522 #endif
523 code_gen_buffer = mmap(start, code_gen_buffer_size,
524 PROT_WRITE | PROT_READ | PROT_EXEC,
525 flags, -1, 0);
526 if (code_gen_buffer == MAP_FAILED) {
527 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
528 exit(1);
531 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
533 int flags;
534 void *addr = NULL;
535 flags = MAP_PRIVATE | MAP_ANONYMOUS;
536 #if defined(__x86_64__)
537 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
538 * 0x40000000 is free */
539 flags |= MAP_FIXED;
540 addr = (void *)0x40000000;
541 /* Cannot map more than that */
542 if (code_gen_buffer_size > (800 * 1024 * 1024))
543 code_gen_buffer_size = (800 * 1024 * 1024);
544 #endif
545 code_gen_buffer = mmap(addr, code_gen_buffer_size,
546 PROT_WRITE | PROT_READ | PROT_EXEC,
547 flags, -1, 0);
548 if (code_gen_buffer == MAP_FAILED) {
549 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
550 exit(1);
553 #else
554 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
555 map_exec(code_gen_buffer, code_gen_buffer_size);
556 #endif
557 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
558 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
559 code_gen_buffer_max_size = code_gen_buffer_size -
560 code_gen_max_block_size();
561 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
562 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
565 /* Must be called before using the QEMU cpus. 'tb_size' is the size
566 (in bytes) allocated to the translation buffer. Zero means default
567 size. */
568 void cpu_exec_init_all(unsigned long tb_size)
570 cpu_gen_init();
571 code_gen_alloc(tb_size);
572 code_gen_ptr = code_gen_buffer;
573 page_init();
574 #if !defined(CONFIG_USER_ONLY)
575 io_mem_init();
576 #endif
577 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
578 /* There's no guest base to take into account, so go ahead and
579 initialize the prologue now. */
580 tcg_prologue_init(&tcg_ctx);
581 #endif
584 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
586 static int cpu_common_post_load(void *opaque, int version_id)
588 CPUState *env = opaque;
590 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
591 version_id is increased. */
592 env->interrupt_request &= ~0x01;
593 tlb_flush(env, 1);
595 return 0;
598 static const VMStateDescription vmstate_cpu_common = {
599 .name = "cpu_common",
600 .version_id = 1,
601 .minimum_version_id = 1,
602 .minimum_version_id_old = 1,
603 .post_load = cpu_common_post_load,
604 .fields = (VMStateField []) {
605 VMSTATE_UINT32(halted, CPUState),
606 VMSTATE_UINT32(interrupt_request, CPUState),
607 VMSTATE_END_OF_LIST()
610 #endif
612 CPUState *qemu_get_cpu(int cpu)
614 CPUState *env = first_cpu;
616 while (env) {
617 if (env->cpu_index == cpu)
618 break;
619 env = env->next_cpu;
622 return env;
625 void cpu_exec_init(CPUState *env)
627 CPUState **penv;
628 int cpu_index;
630 #if defined(CONFIG_USER_ONLY)
631 cpu_list_lock();
632 #endif
633 env->next_cpu = NULL;
634 penv = &first_cpu;
635 cpu_index = 0;
636 while (*penv != NULL) {
637 penv = &(*penv)->next_cpu;
638 cpu_index++;
640 env->cpu_index = cpu_index;
641 env->numa_node = 0;
642 QTAILQ_INIT(&env->breakpoints);
643 QTAILQ_INIT(&env->watchpoints);
644 *penv = env;
645 #if defined(CONFIG_USER_ONLY)
646 cpu_list_unlock();
647 #endif
648 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
649 vmstate_register(cpu_index, &vmstate_cpu_common, env);
650 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
651 cpu_save, cpu_load, env);
652 #endif
655 static inline void invalidate_page_bitmap(PageDesc *p)
657 if (p->code_bitmap) {
658 qemu_free(p->code_bitmap);
659 p->code_bitmap = NULL;
661 p->code_write_count = 0;
664 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
666 static void page_flush_tb_1 (int level, void **lp)
668 int i;
670 if (*lp == NULL) {
671 return;
673 if (level == 0) {
674 PageDesc *pd = *lp;
675 for (i = 0; i < L2_SIZE; ++i) {
676 pd[i].first_tb = NULL;
677 invalidate_page_bitmap(pd + i);
679 } else {
680 void **pp = *lp;
681 for (i = 0; i < L2_SIZE; ++i) {
682 page_flush_tb_1 (level - 1, pp + i);
687 static void page_flush_tb(void)
689 int i;
690 for (i = 0; i < V_L1_SIZE; i++) {
691 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
695 /* flush all the translation blocks */
696 /* XXX: tb_flush is currently not thread safe */
697 void tb_flush(CPUState *env1)
699 CPUState *env;
700 #if defined(DEBUG_FLUSH)
701 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
702 (unsigned long)(code_gen_ptr - code_gen_buffer),
703 nb_tbs, nb_tbs > 0 ?
704 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
705 #endif
706 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
707 cpu_abort(env1, "Internal error: code buffer overflow\n");
709 nb_tbs = 0;
711 for(env = first_cpu; env != NULL; env = env->next_cpu) {
712 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
715 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
716 page_flush_tb();
718 code_gen_ptr = code_gen_buffer;
719 /* XXX: flush processor icache at this point if cache flush is
720 expensive */
721 tb_flush_count++;
724 #ifdef DEBUG_TB_CHECK
726 static void tb_invalidate_check(target_ulong address)
728 TranslationBlock *tb;
729 int i;
730 address &= TARGET_PAGE_MASK;
731 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
732 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
733 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
734 address >= tb->pc + tb->size)) {
735 printf("ERROR invalidate: address=" TARGET_FMT_lx
736 " PC=%08lx size=%04x\n",
737 address, (long)tb->pc, tb->size);
743 /* verify that all the pages have correct rights for code */
744 static void tb_page_check(void)
746 TranslationBlock *tb;
747 int i, flags1, flags2;
749 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
750 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
751 flags1 = page_get_flags(tb->pc);
752 flags2 = page_get_flags(tb->pc + tb->size - 1);
753 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
754 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
755 (long)tb->pc, tb->size, flags1, flags2);
761 #endif
763 /* invalidate one TB */
764 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
765 int next_offset)
767 TranslationBlock *tb1;
768 for(;;) {
769 tb1 = *ptb;
770 if (tb1 == tb) {
771 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
772 break;
774 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
778 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
780 TranslationBlock *tb1;
781 unsigned int n1;
783 for(;;) {
784 tb1 = *ptb;
785 n1 = (long)tb1 & 3;
786 tb1 = (TranslationBlock *)((long)tb1 & ~3);
787 if (tb1 == tb) {
788 *ptb = tb1->page_next[n1];
789 break;
791 ptb = &tb1->page_next[n1];
795 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
797 TranslationBlock *tb1, **ptb;
798 unsigned int n1;
800 ptb = &tb->jmp_next[n];
801 tb1 = *ptb;
802 if (tb1) {
803 /* find tb(n) in circular list */
804 for(;;) {
805 tb1 = *ptb;
806 n1 = (long)tb1 & 3;
807 tb1 = (TranslationBlock *)((long)tb1 & ~3);
808 if (n1 == n && tb1 == tb)
809 break;
810 if (n1 == 2) {
811 ptb = &tb1->jmp_first;
812 } else {
813 ptb = &tb1->jmp_next[n1];
816 /* now we can suppress tb(n) from the list */
817 *ptb = tb->jmp_next[n];
819 tb->jmp_next[n] = NULL;
823 /* reset the jump entry 'n' of a TB so that it is not chained to
824 another TB */
825 static inline void tb_reset_jump(TranslationBlock *tb, int n)
827 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
830 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
832 CPUState *env;
833 PageDesc *p;
834 unsigned int h, n1;
835 tb_page_addr_t phys_pc;
836 TranslationBlock *tb1, *tb2;
838 /* remove the TB from the hash list */
839 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
840 h = tb_phys_hash_func(phys_pc);
841 tb_remove(&tb_phys_hash[h], tb,
842 offsetof(TranslationBlock, phys_hash_next));
844 /* remove the TB from the page list */
845 if (tb->page_addr[0] != page_addr) {
846 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
847 tb_page_remove(&p->first_tb, tb);
848 invalidate_page_bitmap(p);
850 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
851 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
852 tb_page_remove(&p->first_tb, tb);
853 invalidate_page_bitmap(p);
856 tb_invalidated_flag = 1;
858 /* remove the TB from the hash list */
859 h = tb_jmp_cache_hash_func(tb->pc);
860 for(env = first_cpu; env != NULL; env = env->next_cpu) {
861 if (env->tb_jmp_cache[h] == tb)
862 env->tb_jmp_cache[h] = NULL;
865 /* suppress this TB from the two jump lists */
866 tb_jmp_remove(tb, 0);
867 tb_jmp_remove(tb, 1);
869 /* suppress any remaining jumps to this TB */
870 tb1 = tb->jmp_first;
871 for(;;) {
872 n1 = (long)tb1 & 3;
873 if (n1 == 2)
874 break;
875 tb1 = (TranslationBlock *)((long)tb1 & ~3);
876 tb2 = tb1->jmp_next[n1];
877 tb_reset_jump(tb1, n1);
878 tb1->jmp_next[n1] = NULL;
879 tb1 = tb2;
881 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
883 tb_phys_invalidate_count++;
886 static inline void set_bits(uint8_t *tab, int start, int len)
888 int end, mask, end1;
890 end = start + len;
891 tab += start >> 3;
892 mask = 0xff << (start & 7);
893 if ((start & ~7) == (end & ~7)) {
894 if (start < end) {
895 mask &= ~(0xff << (end & 7));
896 *tab |= mask;
898 } else {
899 *tab++ |= mask;
900 start = (start + 8) & ~7;
901 end1 = end & ~7;
902 while (start < end1) {
903 *tab++ = 0xff;
904 start += 8;
906 if (start < end) {
907 mask = ~(0xff << (end & 7));
908 *tab |= mask;
913 static void build_page_bitmap(PageDesc *p)
915 int n, tb_start, tb_end;
916 TranslationBlock *tb;
918 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
920 tb = p->first_tb;
921 while (tb != NULL) {
922 n = (long)tb & 3;
923 tb = (TranslationBlock *)((long)tb & ~3);
924 /* NOTE: this is subtle as a TB may span two physical pages */
925 if (n == 0) {
926 /* NOTE: tb_end may be after the end of the page, but
927 it is not a problem */
928 tb_start = tb->pc & ~TARGET_PAGE_MASK;
929 tb_end = tb_start + tb->size;
930 if (tb_end > TARGET_PAGE_SIZE)
931 tb_end = TARGET_PAGE_SIZE;
932 } else {
933 tb_start = 0;
934 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
936 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
937 tb = tb->page_next[n];
941 TranslationBlock *tb_gen_code(CPUState *env,
942 target_ulong pc, target_ulong cs_base,
943 int flags, int cflags)
945 TranslationBlock *tb;
946 uint8_t *tc_ptr;
947 tb_page_addr_t phys_pc, phys_page2;
948 target_ulong virt_page2;
949 int code_gen_size;
951 phys_pc = get_page_addr_code(env, pc);
952 tb = tb_alloc(pc);
953 if (!tb) {
954 /* flush must be done */
955 tb_flush(env);
956 /* cannot fail at this point */
957 tb = tb_alloc(pc);
958 /* Don't forget to invalidate previous TB info. */
959 tb_invalidated_flag = 1;
961 tc_ptr = code_gen_ptr;
962 tb->tc_ptr = tc_ptr;
963 tb->cs_base = cs_base;
964 tb->flags = flags;
965 tb->cflags = cflags;
966 cpu_gen_code(env, tb, &code_gen_size);
967 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
969 /* check next page if needed */
970 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
971 phys_page2 = -1;
972 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
973 phys_page2 = get_page_addr_code(env, virt_page2);
975 tb_link_page(tb, phys_pc, phys_page2);
976 return tb;
979 /* invalidate all TBs which intersect with the target physical page
980 starting in range [start;end[. NOTE: start and end must refer to
981 the same physical page. 'is_cpu_write_access' should be true if called
982 from a real cpu write access: the virtual CPU will exit the current
983 TB if code is modified inside this TB. */
984 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
985 int is_cpu_write_access)
987 TranslationBlock *tb, *tb_next, *saved_tb;
988 CPUState *env = cpu_single_env;
989 tb_page_addr_t tb_start, tb_end;
990 PageDesc *p;
991 int n;
992 #ifdef TARGET_HAS_PRECISE_SMC
993 int current_tb_not_found = is_cpu_write_access;
994 TranslationBlock *current_tb = NULL;
995 int current_tb_modified = 0;
996 target_ulong current_pc = 0;
997 target_ulong current_cs_base = 0;
998 int current_flags = 0;
999 #endif /* TARGET_HAS_PRECISE_SMC */
1001 p = page_find(start >> TARGET_PAGE_BITS);
1002 if (!p)
1003 return;
1004 if (!p->code_bitmap &&
1005 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1006 is_cpu_write_access) {
1007 /* build code bitmap */
1008 build_page_bitmap(p);
1011 /* we remove all the TBs in the range [start, end[ */
1012 /* XXX: see if in some cases it could be faster to invalidate all the code */
1013 tb = p->first_tb;
1014 while (tb != NULL) {
1015 n = (long)tb & 3;
1016 tb = (TranslationBlock *)((long)tb & ~3);
1017 tb_next = tb->page_next[n];
1018 /* NOTE: this is subtle as a TB may span two physical pages */
1019 if (n == 0) {
1020 /* NOTE: tb_end may be after the end of the page, but
1021 it is not a problem */
1022 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1023 tb_end = tb_start + tb->size;
1024 } else {
1025 tb_start = tb->page_addr[1];
1026 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1028 if (!(tb_end <= start || tb_start >= end)) {
1029 #ifdef TARGET_HAS_PRECISE_SMC
1030 if (current_tb_not_found) {
1031 current_tb_not_found = 0;
1032 current_tb = NULL;
1033 if (env->mem_io_pc) {
1034 /* now we have a real cpu fault */
1035 current_tb = tb_find_pc(env->mem_io_pc);
1038 if (current_tb == tb &&
1039 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1040 /* If we are modifying the current TB, we must stop
1041 its execution. We could be more precise by checking
1042 that the modification is after the current PC, but it
1043 would require a specialized function to partially
1044 restore the CPU state */
1046 current_tb_modified = 1;
1047 cpu_restore_state(current_tb, env,
1048 env->mem_io_pc, NULL);
1049 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1050 &current_flags);
1052 #endif /* TARGET_HAS_PRECISE_SMC */
1053 /* we need to do that to handle the case where a signal
1054 occurs while doing tb_phys_invalidate() */
1055 saved_tb = NULL;
1056 if (env) {
1057 saved_tb = env->current_tb;
1058 env->current_tb = NULL;
1060 tb_phys_invalidate(tb, -1);
1061 if (env) {
1062 env->current_tb = saved_tb;
1063 if (env->interrupt_request && env->current_tb)
1064 cpu_interrupt(env, env->interrupt_request);
1067 tb = tb_next;
1069 #if !defined(CONFIG_USER_ONLY)
1070 /* if no code remaining, no need to continue to use slow writes */
1071 if (!p->first_tb) {
1072 invalidate_page_bitmap(p);
1073 if (is_cpu_write_access) {
1074 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1077 #endif
1078 #ifdef TARGET_HAS_PRECISE_SMC
1079 if (current_tb_modified) {
1080 /* we generate a block containing just the instruction
1081 modifying the memory. It will ensure that it cannot modify
1082 itself */
1083 env->current_tb = NULL;
1084 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1085 cpu_resume_from_signal(env, NULL);
1087 #endif
1090 /* len must be <= 8 and start must be a multiple of len */
1091 static inline void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1093 PageDesc *p;
1094 int offset, b;
1095 #if 0
1096 if (1) {
1097 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1098 cpu_single_env->mem_io_vaddr, len,
1099 cpu_single_env->eip,
1100 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1102 #endif
1103 p = page_find(start >> TARGET_PAGE_BITS);
1104 if (!p)
1105 return;
1106 if (p->code_bitmap) {
1107 offset = start & ~TARGET_PAGE_MASK;
1108 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1109 if (b & ((1 << len) - 1))
1110 goto do_invalidate;
1111 } else {
1112 do_invalidate:
1113 tb_invalidate_phys_page_range(start, start + len, 1);
1117 #if !defined(CONFIG_SOFTMMU)
1118 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1119 unsigned long pc, void *puc)
1121 TranslationBlock *tb;
1122 PageDesc *p;
1123 int n;
1124 #ifdef TARGET_HAS_PRECISE_SMC
1125 TranslationBlock *current_tb = NULL;
1126 CPUState *env = cpu_single_env;
1127 int current_tb_modified = 0;
1128 target_ulong current_pc = 0;
1129 target_ulong current_cs_base = 0;
1130 int current_flags = 0;
1131 #endif
1133 addr &= TARGET_PAGE_MASK;
1134 p = page_find(addr >> TARGET_PAGE_BITS);
1135 if (!p)
1136 return;
1137 tb = p->first_tb;
1138 #ifdef TARGET_HAS_PRECISE_SMC
1139 if (tb && pc != 0) {
1140 current_tb = tb_find_pc(pc);
1142 #endif
1143 while (tb != NULL) {
1144 n = (long)tb & 3;
1145 tb = (TranslationBlock *)((long)tb & ~3);
1146 #ifdef TARGET_HAS_PRECISE_SMC
1147 if (current_tb == tb &&
1148 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1149 /* If we are modifying the current TB, we must stop
1150 its execution. We could be more precise by checking
1151 that the modification is after the current PC, but it
1152 would require a specialized function to partially
1153 restore the CPU state */
1155 current_tb_modified = 1;
1156 cpu_restore_state(current_tb, env, pc, puc);
1157 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1158 &current_flags);
1160 #endif /* TARGET_HAS_PRECISE_SMC */
1161 tb_phys_invalidate(tb, addr);
1162 tb = tb->page_next[n];
1164 p->first_tb = NULL;
1165 #ifdef TARGET_HAS_PRECISE_SMC
1166 if (current_tb_modified) {
1167 /* we generate a block containing just the instruction
1168 modifying the memory. It will ensure that it cannot modify
1169 itself */
1170 env->current_tb = NULL;
1171 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1172 cpu_resume_from_signal(env, puc);
1174 #endif
1176 #endif
1178 /* add the tb in the target page and protect it if necessary */
1179 static inline void tb_alloc_page(TranslationBlock *tb,
1180 unsigned int n, tb_page_addr_t page_addr)
1182 PageDesc *p;
1183 TranslationBlock *last_first_tb;
1185 tb->page_addr[n] = page_addr;
1186 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1187 tb->page_next[n] = p->first_tb;
1188 last_first_tb = p->first_tb;
1189 p->first_tb = (TranslationBlock *)((long)tb | n);
1190 invalidate_page_bitmap(p);
1192 #if defined(TARGET_HAS_SMC) || 1
1194 #if defined(CONFIG_USER_ONLY)
1195 if (p->flags & PAGE_WRITE) {
1196 target_ulong addr;
1197 PageDesc *p2;
1198 int prot;
1200 /* force the host page as non writable (writes will have a
1201 page fault + mprotect overhead) */
1202 page_addr &= qemu_host_page_mask;
1203 prot = 0;
1204 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1205 addr += TARGET_PAGE_SIZE) {
1207 p2 = page_find (addr >> TARGET_PAGE_BITS);
1208 if (!p2)
1209 continue;
1210 prot |= p2->flags;
1211 p2->flags &= ~PAGE_WRITE;
1213 mprotect(g2h(page_addr), qemu_host_page_size,
1214 (prot & PAGE_BITS) & ~PAGE_WRITE);
1215 #ifdef DEBUG_TB_INVALIDATE
1216 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1217 page_addr);
1218 #endif
1220 #else
1221 /* if some code is already present, then the pages are already
1222 protected. So we handle the case where only the first TB is
1223 allocated in a physical page */
1224 if (!last_first_tb) {
1225 tlb_protect_code(page_addr);
1227 #endif
1229 #endif /* TARGET_HAS_SMC */
1232 /* Allocate a new translation block. Flush the translation buffer if
1233 too many translation blocks or too much generated code. */
1234 TranslationBlock *tb_alloc(target_ulong pc)
1236 TranslationBlock *tb;
1238 if (nb_tbs >= code_gen_max_blocks ||
1239 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1240 return NULL;
1241 tb = &tbs[nb_tbs++];
1242 tb->pc = pc;
1243 tb->cflags = 0;
1244 return tb;
1247 void tb_free(TranslationBlock *tb)
1249 /* In practice this is mostly used for single use temporary TB
1250 Ignore the hard cases and just back up if this TB happens to
1251 be the last one generated. */
1252 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1253 code_gen_ptr = tb->tc_ptr;
1254 nb_tbs--;
1258 /* add a new TB and link it to the physical page tables. phys_page2 is
1259 (-1) to indicate that only one page contains the TB. */
1260 void tb_link_page(TranslationBlock *tb,
1261 tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
1263 unsigned int h;
1264 TranslationBlock **ptb;
1266 /* Grab the mmap lock to stop another thread invalidating this TB
1267 before we are done. */
1268 mmap_lock();
1269 /* add in the physical hash table */
1270 h = tb_phys_hash_func(phys_pc);
1271 ptb = &tb_phys_hash[h];
1272 tb->phys_hash_next = *ptb;
1273 *ptb = tb;
1275 /* add in the page list */
1276 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1277 if (phys_page2 != -1)
1278 tb_alloc_page(tb, 1, phys_page2);
1279 else
1280 tb->page_addr[1] = -1;
1282 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1283 tb->jmp_next[0] = NULL;
1284 tb->jmp_next[1] = NULL;
1286 /* init original jump addresses */
1287 if (tb->tb_next_offset[0] != 0xffff)
1288 tb_reset_jump(tb, 0);
1289 if (tb->tb_next_offset[1] != 0xffff)
1290 tb_reset_jump(tb, 1);
1292 #ifdef DEBUG_TB_CHECK
1293 tb_page_check();
1294 #endif
1295 mmap_unlock();
1298 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1299 tb[1].tc_ptr. Return NULL if not found */
1300 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1302 int m_min, m_max, m;
1303 unsigned long v;
1304 TranslationBlock *tb;
1306 if (nb_tbs <= 0)
1307 return NULL;
1308 if (tc_ptr < (unsigned long)code_gen_buffer ||
1309 tc_ptr >= (unsigned long)code_gen_ptr)
1310 return NULL;
1311 /* binary search (cf Knuth) */
1312 m_min = 0;
1313 m_max = nb_tbs - 1;
1314 while (m_min <= m_max) {
1315 m = (m_min + m_max) >> 1;
1316 tb = &tbs[m];
1317 v = (unsigned long)tb->tc_ptr;
1318 if (v == tc_ptr)
1319 return tb;
1320 else if (tc_ptr < v) {
1321 m_max = m - 1;
1322 } else {
1323 m_min = m + 1;
1326 return &tbs[m_max];
1329 static void tb_reset_jump_recursive(TranslationBlock *tb);
1331 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1333 TranslationBlock *tb1, *tb_next, **ptb;
1334 unsigned int n1;
1336 tb1 = tb->jmp_next[n];
1337 if (tb1 != NULL) {
1338 /* find head of list */
1339 for(;;) {
1340 n1 = (long)tb1 & 3;
1341 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1342 if (n1 == 2)
1343 break;
1344 tb1 = tb1->jmp_next[n1];
1346 /* we are now sure now that tb jumps to tb1 */
1347 tb_next = tb1;
1349 /* remove tb from the jmp_first list */
1350 ptb = &tb_next->jmp_first;
1351 for(;;) {
1352 tb1 = *ptb;
1353 n1 = (long)tb1 & 3;
1354 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1355 if (n1 == n && tb1 == tb)
1356 break;
1357 ptb = &tb1->jmp_next[n1];
1359 *ptb = tb->jmp_next[n];
1360 tb->jmp_next[n] = NULL;
1362 /* suppress the jump to next tb in generated code */
1363 tb_reset_jump(tb, n);
1365 /* suppress jumps in the tb on which we could have jumped */
1366 tb_reset_jump_recursive(tb_next);
1370 static void tb_reset_jump_recursive(TranslationBlock *tb)
1372 tb_reset_jump_recursive2(tb, 0);
1373 tb_reset_jump_recursive2(tb, 1);
1376 #if defined(TARGET_HAS_ICE)
1377 #if defined(CONFIG_USER_ONLY)
1378 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1380 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1382 #else
1383 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1385 target_phys_addr_t addr;
1386 target_ulong pd;
1387 ram_addr_t ram_addr;
1388 PhysPageDesc *p;
1390 addr = cpu_get_phys_page_debug(env, pc);
1391 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1392 if (!p) {
1393 pd = IO_MEM_UNASSIGNED;
1394 } else {
1395 pd = p->phys_offset;
1397 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1398 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1400 #endif
1401 #endif /* TARGET_HAS_ICE */
1403 #if defined(CONFIG_USER_ONLY)
1404 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1409 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1410 int flags, CPUWatchpoint **watchpoint)
1412 return -ENOSYS;
1414 #else
1415 /* Add a watchpoint. */
1416 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1417 int flags, CPUWatchpoint **watchpoint)
1419 target_ulong len_mask = ~(len - 1);
1420 CPUWatchpoint *wp;
1422 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1423 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1424 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1425 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1426 return -EINVAL;
1428 wp = qemu_malloc(sizeof(*wp));
1430 wp->vaddr = addr;
1431 wp->len_mask = len_mask;
1432 wp->flags = flags;
1434 /* keep all GDB-injected watchpoints in front */
1435 if (flags & BP_GDB)
1436 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1437 else
1438 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1440 tlb_flush_page(env, addr);
1442 if (watchpoint)
1443 *watchpoint = wp;
1444 return 0;
1447 /* Remove a specific watchpoint. */
1448 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1449 int flags)
1451 target_ulong len_mask = ~(len - 1);
1452 CPUWatchpoint *wp;
1454 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1455 if (addr == wp->vaddr && len_mask == wp->len_mask
1456 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1457 cpu_watchpoint_remove_by_ref(env, wp);
1458 return 0;
1461 return -ENOENT;
1464 /* Remove a specific watchpoint by reference. */
1465 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1467 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1469 tlb_flush_page(env, watchpoint->vaddr);
1471 qemu_free(watchpoint);
1474 /* Remove all matching watchpoints. */
1475 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1477 CPUWatchpoint *wp, *next;
1479 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1480 if (wp->flags & mask)
1481 cpu_watchpoint_remove_by_ref(env, wp);
1484 #endif
1486 /* Add a breakpoint. */
1487 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1488 CPUBreakpoint **breakpoint)
1490 #if defined(TARGET_HAS_ICE)
1491 CPUBreakpoint *bp;
1493 bp = qemu_malloc(sizeof(*bp));
1495 bp->pc = pc;
1496 bp->flags = flags;
1498 /* keep all GDB-injected breakpoints in front */
1499 if (flags & BP_GDB)
1500 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1501 else
1502 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1504 breakpoint_invalidate(env, pc);
1506 if (breakpoint)
1507 *breakpoint = bp;
1508 return 0;
1509 #else
1510 return -ENOSYS;
1511 #endif
1514 /* Remove a specific breakpoint. */
1515 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1517 #if defined(TARGET_HAS_ICE)
1518 CPUBreakpoint *bp;
1520 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1521 if (bp->pc == pc && bp->flags == flags) {
1522 cpu_breakpoint_remove_by_ref(env, bp);
1523 return 0;
1526 return -ENOENT;
1527 #else
1528 return -ENOSYS;
1529 #endif
1532 /* Remove a specific breakpoint by reference. */
1533 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1535 #if defined(TARGET_HAS_ICE)
1536 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1538 breakpoint_invalidate(env, breakpoint->pc);
1540 qemu_free(breakpoint);
1541 #endif
1544 /* Remove all matching breakpoints. */
1545 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1547 #if defined(TARGET_HAS_ICE)
1548 CPUBreakpoint *bp, *next;
1550 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1551 if (bp->flags & mask)
1552 cpu_breakpoint_remove_by_ref(env, bp);
1554 #endif
1557 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1558 CPU loop after each instruction */
1559 void cpu_single_step(CPUState *env, int enabled)
1561 #if defined(TARGET_HAS_ICE)
1562 if (env->singlestep_enabled != enabled) {
1563 env->singlestep_enabled = enabled;
1564 if (kvm_enabled())
1565 kvm_update_guest_debug(env, 0);
1566 else {
1567 /* must flush all the translated code to avoid inconsistencies */
1568 /* XXX: only flush what is necessary */
1569 tb_flush(env);
1572 #endif
1575 /* enable or disable low levels log */
1576 void cpu_set_log(int log_flags)
1578 loglevel = log_flags;
1579 if (loglevel && !logfile) {
1580 logfile = fopen(logfilename, log_append ? "a" : "w");
1581 if (!logfile) {
1582 perror(logfilename);
1583 _exit(1);
1585 #if !defined(CONFIG_SOFTMMU)
1586 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1588 static char logfile_buf[4096];
1589 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1591 #elif !defined(_WIN32)
1592 /* Win32 doesn't support line-buffering and requires size >= 2 */
1593 setvbuf(logfile, NULL, _IOLBF, 0);
1594 #endif
1595 log_append = 1;
1597 if (!loglevel && logfile) {
1598 fclose(logfile);
1599 logfile = NULL;
1603 void cpu_set_log_filename(const char *filename)
1605 logfilename = strdup(filename);
1606 if (logfile) {
1607 fclose(logfile);
1608 logfile = NULL;
1610 cpu_set_log(loglevel);
1613 static void cpu_unlink_tb(CPUState *env)
1615 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1616 problem and hope the cpu will stop of its own accord. For userspace
1617 emulation this often isn't actually as bad as it sounds. Often
1618 signals are used primarily to interrupt blocking syscalls. */
1619 TranslationBlock *tb;
1620 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1622 spin_lock(&interrupt_lock);
1623 tb = env->current_tb;
1624 /* if the cpu is currently executing code, we must unlink it and
1625 all the potentially executing TB */
1626 if (tb) {
1627 env->current_tb = NULL;
1628 tb_reset_jump_recursive(tb);
1630 spin_unlock(&interrupt_lock);
1633 /* mask must never be zero, except for A20 change call */
1634 void cpu_interrupt(CPUState *env, int mask)
1636 int old_mask;
1638 old_mask = env->interrupt_request;
1639 env->interrupt_request |= mask;
1641 #ifndef CONFIG_USER_ONLY
1643 * If called from iothread context, wake the target cpu in
1644 * case its halted.
1646 if (!qemu_cpu_self(env)) {
1647 qemu_cpu_kick(env);
1648 return;
1650 #endif
1652 if (use_icount) {
1653 env->icount_decr.u16.high = 0xffff;
1654 #ifndef CONFIG_USER_ONLY
1655 if (!can_do_io(env)
1656 && (mask & ~old_mask) != 0) {
1657 cpu_abort(env, "Raised interrupt while not in I/O function");
1659 #endif
1660 } else {
1661 cpu_unlink_tb(env);
1665 void cpu_reset_interrupt(CPUState *env, int mask)
1667 env->interrupt_request &= ~mask;
1670 void cpu_exit(CPUState *env)
1672 env->exit_request = 1;
1673 cpu_unlink_tb(env);
1676 const CPULogItem cpu_log_items[] = {
1677 { CPU_LOG_TB_OUT_ASM, "out_asm",
1678 "show generated host assembly code for each compiled TB" },
1679 { CPU_LOG_TB_IN_ASM, "in_asm",
1680 "show target assembly code for each compiled TB" },
1681 { CPU_LOG_TB_OP, "op",
1682 "show micro ops for each compiled TB" },
1683 { CPU_LOG_TB_OP_OPT, "op_opt",
1684 "show micro ops "
1685 #ifdef TARGET_I386
1686 "before eflags optimization and "
1687 #endif
1688 "after liveness analysis" },
1689 { CPU_LOG_INT, "int",
1690 "show interrupts/exceptions in short format" },
1691 { CPU_LOG_EXEC, "exec",
1692 "show trace before each executed TB (lots of logs)" },
1693 { CPU_LOG_TB_CPU, "cpu",
1694 "show CPU state before block translation" },
1695 #ifdef TARGET_I386
1696 { CPU_LOG_PCALL, "pcall",
1697 "show protected mode far calls/returns/exceptions" },
1698 { CPU_LOG_RESET, "cpu_reset",
1699 "show CPU state before CPU resets" },
1700 #endif
1701 #ifdef DEBUG_IOPORT
1702 { CPU_LOG_IOPORT, "ioport",
1703 "show all i/o ports accesses" },
1704 #endif
1705 { 0, NULL, NULL },
1708 #ifndef CONFIG_USER_ONLY
1709 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1710 = QLIST_HEAD_INITIALIZER(memory_client_list);
1712 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1713 ram_addr_t size,
1714 ram_addr_t phys_offset)
1716 CPUPhysMemoryClient *client;
1717 QLIST_FOREACH(client, &memory_client_list, list) {
1718 client->set_memory(client, start_addr, size, phys_offset);
1722 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1723 target_phys_addr_t end)
1725 CPUPhysMemoryClient *client;
1726 QLIST_FOREACH(client, &memory_client_list, list) {
1727 int r = client->sync_dirty_bitmap(client, start, end);
1728 if (r < 0)
1729 return r;
1731 return 0;
1734 static int cpu_notify_migration_log(int enable)
1736 CPUPhysMemoryClient *client;
1737 QLIST_FOREACH(client, &memory_client_list, list) {
1738 int r = client->migration_log(client, enable);
1739 if (r < 0)
1740 return r;
1742 return 0;
1745 static void phys_page_for_each_1(CPUPhysMemoryClient *client,
1746 int level, void **lp)
1748 int i;
1750 if (*lp == NULL) {
1751 return;
1753 if (level == 0) {
1754 PhysPageDesc *pd = *lp;
1755 for (i = 0; i < L2_SIZE; ++i) {
1756 if (pd[i].phys_offset != IO_MEM_UNASSIGNED) {
1757 client->set_memory(client, pd[i].region_offset,
1758 TARGET_PAGE_SIZE, pd[i].phys_offset);
1761 } else {
1762 void **pp = *lp;
1763 for (i = 0; i < L2_SIZE; ++i) {
1764 phys_page_for_each_1(client, level - 1, pp + i);
1769 static void phys_page_for_each(CPUPhysMemoryClient *client)
1771 int i;
1772 for (i = 0; i < P_L1_SIZE; ++i) {
1773 phys_page_for_each_1(client, P_L1_SHIFT / L2_BITS - 1,
1774 l1_phys_map + 1);
1778 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1780 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1781 phys_page_for_each(client);
1784 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1786 QLIST_REMOVE(client, list);
1788 #endif
1790 static int cmp1(const char *s1, int n, const char *s2)
1792 if (strlen(s2) != n)
1793 return 0;
1794 return memcmp(s1, s2, n) == 0;
1797 /* takes a comma separated list of log masks. Return 0 if error. */
1798 int cpu_str_to_log_mask(const char *str)
1800 const CPULogItem *item;
1801 int mask;
1802 const char *p, *p1;
1804 p = str;
1805 mask = 0;
1806 for(;;) {
1807 p1 = strchr(p, ',');
1808 if (!p1)
1809 p1 = p + strlen(p);
1810 if(cmp1(p,p1-p,"all")) {
1811 for(item = cpu_log_items; item->mask != 0; item++) {
1812 mask |= item->mask;
1814 } else {
1815 for(item = cpu_log_items; item->mask != 0; item++) {
1816 if (cmp1(p, p1 - p, item->name))
1817 goto found;
1819 return 0;
1821 found:
1822 mask |= item->mask;
1823 if (*p1 != ',')
1824 break;
1825 p = p1 + 1;
1827 return mask;
1830 void cpu_abort(CPUState *env, const char *fmt, ...)
1832 va_list ap;
1833 va_list ap2;
1835 va_start(ap, fmt);
1836 va_copy(ap2, ap);
1837 fprintf(stderr, "qemu: fatal: ");
1838 vfprintf(stderr, fmt, ap);
1839 fprintf(stderr, "\n");
1840 #ifdef TARGET_I386
1841 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1842 #else
1843 cpu_dump_state(env, stderr, fprintf, 0);
1844 #endif
1845 if (qemu_log_enabled()) {
1846 qemu_log("qemu: fatal: ");
1847 qemu_log_vprintf(fmt, ap2);
1848 qemu_log("\n");
1849 #ifdef TARGET_I386
1850 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1851 #else
1852 log_cpu_state(env, 0);
1853 #endif
1854 qemu_log_flush();
1855 qemu_log_close();
1857 va_end(ap2);
1858 va_end(ap);
1859 #if defined(CONFIG_USER_ONLY)
1861 struct sigaction act;
1862 sigfillset(&act.sa_mask);
1863 act.sa_handler = SIG_DFL;
1864 sigaction(SIGABRT, &act, NULL);
1866 #endif
1867 abort();
1870 CPUState *cpu_copy(CPUState *env)
1872 CPUState *new_env = cpu_init(env->cpu_model_str);
1873 CPUState *next_cpu = new_env->next_cpu;
1874 int cpu_index = new_env->cpu_index;
1875 #if defined(TARGET_HAS_ICE)
1876 CPUBreakpoint *bp;
1877 CPUWatchpoint *wp;
1878 #endif
1880 memcpy(new_env, env, sizeof(CPUState));
1882 /* Preserve chaining and index. */
1883 new_env->next_cpu = next_cpu;
1884 new_env->cpu_index = cpu_index;
1886 /* Clone all break/watchpoints.
1887 Note: Once we support ptrace with hw-debug register access, make sure
1888 BP_CPU break/watchpoints are handled correctly on clone. */
1889 QTAILQ_INIT(&env->breakpoints);
1890 QTAILQ_INIT(&env->watchpoints);
1891 #if defined(TARGET_HAS_ICE)
1892 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1893 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1895 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1896 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1897 wp->flags, NULL);
1899 #endif
1901 return new_env;
1904 #if !defined(CONFIG_USER_ONLY)
1906 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1908 unsigned int i;
1910 /* Discard jump cache entries for any tb which might potentially
1911 overlap the flushed page. */
1912 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1913 memset (&env->tb_jmp_cache[i], 0,
1914 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1916 i = tb_jmp_cache_hash_page(addr);
1917 memset (&env->tb_jmp_cache[i], 0,
1918 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1921 static CPUTLBEntry s_cputlb_empty_entry = {
1922 .addr_read = -1,
1923 .addr_write = -1,
1924 .addr_code = -1,
1925 .addend = -1,
1928 /* NOTE: if flush_global is true, also flush global entries (not
1929 implemented yet) */
1930 void tlb_flush(CPUState *env, int flush_global)
1932 int i;
1934 #if defined(DEBUG_TLB)
1935 printf("tlb_flush:\n");
1936 #endif
1937 /* must reset current TB so that interrupts cannot modify the
1938 links while we are modifying them */
1939 env->current_tb = NULL;
1941 for(i = 0; i < CPU_TLB_SIZE; i++) {
1942 int mmu_idx;
1943 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1944 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1948 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1950 env->tlb_flush_addr = -1;
1951 env->tlb_flush_mask = 0;
1952 tlb_flush_count++;
1955 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1957 if (addr == (tlb_entry->addr_read &
1958 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1959 addr == (tlb_entry->addr_write &
1960 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1961 addr == (tlb_entry->addr_code &
1962 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1963 *tlb_entry = s_cputlb_empty_entry;
1967 void tlb_flush_page(CPUState *env, target_ulong addr)
1969 int i;
1970 int mmu_idx;
1972 #if defined(DEBUG_TLB)
1973 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1974 #endif
1975 /* Check if we need to flush due to large pages. */
1976 if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
1977 #if defined(DEBUG_TLB)
1978 printf("tlb_flush_page: forced full flush ("
1979 TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
1980 env->tlb_flush_addr, env->tlb_flush_mask);
1981 #endif
1982 tlb_flush(env, 1);
1983 return;
1985 /* must reset current TB so that interrupts cannot modify the
1986 links while we are modifying them */
1987 env->current_tb = NULL;
1989 addr &= TARGET_PAGE_MASK;
1990 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1991 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1992 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1994 tlb_flush_jmp_cache(env, addr);
1997 /* update the TLBs so that writes to code in the virtual page 'addr'
1998 can be detected */
1999 static void tlb_protect_code(ram_addr_t ram_addr)
2001 cpu_physical_memory_reset_dirty(ram_addr,
2002 ram_addr + TARGET_PAGE_SIZE,
2003 CODE_DIRTY_FLAG);
2006 /* update the TLB so that writes in physical page 'phys_addr' are no longer
2007 tested for self modifying code */
2008 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
2009 target_ulong vaddr)
2011 cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
2014 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
2015 unsigned long start, unsigned long length)
2017 unsigned long addr;
2018 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2019 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
2020 if ((addr - start) < length) {
2021 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
2026 /* Note: start and end must be within the same ram block. */
2027 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
2028 int dirty_flags)
2030 CPUState *env;
2031 unsigned long length, start1;
2032 int i;
2034 start &= TARGET_PAGE_MASK;
2035 end = TARGET_PAGE_ALIGN(end);
2037 length = end - start;
2038 if (length == 0)
2039 return;
2040 cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
2042 /* we modify the TLB cache so that the dirty bit will be set again
2043 when accessing the range */
2044 start1 = (unsigned long)qemu_get_ram_ptr(start);
2045 /* Chek that we don't span multiple blocks - this breaks the
2046 address comparisons below. */
2047 if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
2048 != (end - 1) - start) {
2049 abort();
2052 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2053 int mmu_idx;
2054 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2055 for(i = 0; i < CPU_TLB_SIZE; i++)
2056 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
2057 start1, length);
2062 int cpu_physical_memory_set_dirty_tracking(int enable)
2064 int ret = 0;
2065 in_migration = enable;
2066 ret = cpu_notify_migration_log(!!enable);
2067 return ret;
2070 int cpu_physical_memory_get_dirty_tracking(void)
2072 return in_migration;
2075 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2076 target_phys_addr_t end_addr)
2078 int ret;
2080 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2081 return ret;
2084 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2086 ram_addr_t ram_addr;
2087 void *p;
2089 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2090 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2091 + tlb_entry->addend);
2092 ram_addr = qemu_ram_addr_from_host(p);
2093 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2094 tlb_entry->addr_write |= TLB_NOTDIRTY;
2099 /* update the TLB according to the current state of the dirty bits */
2100 void cpu_tlb_update_dirty(CPUState *env)
2102 int i;
2103 int mmu_idx;
2104 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2105 for(i = 0; i < CPU_TLB_SIZE; i++)
2106 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2110 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2112 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2113 tlb_entry->addr_write = vaddr;
2116 /* update the TLB corresponding to virtual page vaddr
2117 so that it is no longer dirty */
2118 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2120 int i;
2121 int mmu_idx;
2123 vaddr &= TARGET_PAGE_MASK;
2124 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2125 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2126 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2129 /* Our TLB does not support large pages, so remember the area covered by
2130 large pages and trigger a full TLB flush if these are invalidated. */
2131 static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
2132 target_ulong size)
2134 target_ulong mask = ~(size - 1);
2136 if (env->tlb_flush_addr == (target_ulong)-1) {
2137 env->tlb_flush_addr = vaddr & mask;
2138 env->tlb_flush_mask = mask;
2139 return;
2141 /* Extend the existing region to include the new page.
2142 This is a compromise between unnecessary flushes and the cost
2143 of maintaining a full variable size TLB. */
2144 mask &= env->tlb_flush_mask;
2145 while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
2146 mask <<= 1;
2148 env->tlb_flush_addr &= mask;
2149 env->tlb_flush_mask = mask;
2152 /* Add a new TLB entry. At most one entry for a given virtual address
2153 is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
2154 supplied size is only used by tlb_flush_page. */
2155 void tlb_set_page(CPUState *env, target_ulong vaddr,
2156 target_phys_addr_t paddr, int prot,
2157 int mmu_idx, target_ulong size)
2159 PhysPageDesc *p;
2160 unsigned long pd;
2161 unsigned int index;
2162 target_ulong address;
2163 target_ulong code_address;
2164 unsigned long addend;
2165 CPUTLBEntry *te;
2166 CPUWatchpoint *wp;
2167 target_phys_addr_t iotlb;
2169 assert(size >= TARGET_PAGE_SIZE);
2170 if (size != TARGET_PAGE_SIZE) {
2171 tlb_add_large_page(env, vaddr, size);
2173 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2174 if (!p) {
2175 pd = IO_MEM_UNASSIGNED;
2176 } else {
2177 pd = p->phys_offset;
2179 #if defined(DEBUG_TLB)
2180 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2181 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2182 #endif
2184 address = vaddr;
2185 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2186 /* IO memory case (romd handled later) */
2187 address |= TLB_MMIO;
2189 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2190 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2191 /* Normal RAM. */
2192 iotlb = pd & TARGET_PAGE_MASK;
2193 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2194 iotlb |= IO_MEM_NOTDIRTY;
2195 else
2196 iotlb |= IO_MEM_ROM;
2197 } else {
2198 /* IO handlers are currently passed a physical address.
2199 It would be nice to pass an offset from the base address
2200 of that region. This would avoid having to special case RAM,
2201 and avoid full address decoding in every device.
2202 We can't use the high bits of pd for this because
2203 IO_MEM_ROMD uses these as a ram address. */
2204 iotlb = (pd & ~TARGET_PAGE_MASK);
2205 if (p) {
2206 iotlb += p->region_offset;
2207 } else {
2208 iotlb += paddr;
2212 code_address = address;
2213 /* Make accesses to pages with watchpoints go via the
2214 watchpoint trap routines. */
2215 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2216 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2217 iotlb = io_mem_watch + paddr;
2218 /* TODO: The memory case can be optimized by not trapping
2219 reads of pages with a write breakpoint. */
2220 address |= TLB_MMIO;
2224 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2225 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2226 te = &env->tlb_table[mmu_idx][index];
2227 te->addend = addend - vaddr;
2228 if (prot & PAGE_READ) {
2229 te->addr_read = address;
2230 } else {
2231 te->addr_read = -1;
2234 if (prot & PAGE_EXEC) {
2235 te->addr_code = code_address;
2236 } else {
2237 te->addr_code = -1;
2239 if (prot & PAGE_WRITE) {
2240 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2241 (pd & IO_MEM_ROMD)) {
2242 /* Write access calls the I/O callback. */
2243 te->addr_write = address | TLB_MMIO;
2244 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2245 !cpu_physical_memory_is_dirty(pd)) {
2246 te->addr_write = address | TLB_NOTDIRTY;
2247 } else {
2248 te->addr_write = address;
2250 } else {
2251 te->addr_write = -1;
2255 #else
2257 void tlb_flush(CPUState *env, int flush_global)
2261 void tlb_flush_page(CPUState *env, target_ulong addr)
2266 * Walks guest process memory "regions" one by one
2267 * and calls callback function 'fn' for each region.
2270 struct walk_memory_regions_data
2272 walk_memory_regions_fn fn;
2273 void *priv;
2274 unsigned long start;
2275 int prot;
2278 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2279 abi_ulong end, int new_prot)
2281 if (data->start != -1ul) {
2282 int rc = data->fn(data->priv, data->start, end, data->prot);
2283 if (rc != 0) {
2284 return rc;
2288 data->start = (new_prot ? end : -1ul);
2289 data->prot = new_prot;
2291 return 0;
2294 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2295 abi_ulong base, int level, void **lp)
2297 abi_ulong pa;
2298 int i, rc;
2300 if (*lp == NULL) {
2301 return walk_memory_regions_end(data, base, 0);
2304 if (level == 0) {
2305 PageDesc *pd = *lp;
2306 for (i = 0; i < L2_SIZE; ++i) {
2307 int prot = pd[i].flags;
2309 pa = base | (i << TARGET_PAGE_BITS);
2310 if (prot != data->prot) {
2311 rc = walk_memory_regions_end(data, pa, prot);
2312 if (rc != 0) {
2313 return rc;
2317 } else {
2318 void **pp = *lp;
2319 for (i = 0; i < L2_SIZE; ++i) {
2320 pa = base | ((abi_ulong)i <<
2321 (TARGET_PAGE_BITS + L2_BITS * level));
2322 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2323 if (rc != 0) {
2324 return rc;
2329 return 0;
2332 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2334 struct walk_memory_regions_data data;
2335 unsigned long i;
2337 data.fn = fn;
2338 data.priv = priv;
2339 data.start = -1ul;
2340 data.prot = 0;
2342 for (i = 0; i < V_L1_SIZE; i++) {
2343 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
2344 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
2345 if (rc != 0) {
2346 return rc;
2350 return walk_memory_regions_end(&data, 0, 0);
2353 static int dump_region(void *priv, abi_ulong start,
2354 abi_ulong end, unsigned long prot)
2356 FILE *f = (FILE *)priv;
2358 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
2359 " "TARGET_ABI_FMT_lx" %c%c%c\n",
2360 start, end, end - start,
2361 ((prot & PAGE_READ) ? 'r' : '-'),
2362 ((prot & PAGE_WRITE) ? 'w' : '-'),
2363 ((prot & PAGE_EXEC) ? 'x' : '-'));
2365 return (0);
2368 /* dump memory mappings */
2369 void page_dump(FILE *f)
2371 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2372 "start", "end", "size", "prot");
2373 walk_memory_regions(f, dump_region);
2376 int page_get_flags(target_ulong address)
2378 PageDesc *p;
2380 p = page_find(address >> TARGET_PAGE_BITS);
2381 if (!p)
2382 return 0;
2383 return p->flags;
2386 /* Modify the flags of a page and invalidate the code if necessary.
2387 The flag PAGE_WRITE_ORG is positioned automatically depending
2388 on PAGE_WRITE. The mmap_lock should already be held. */
2389 void page_set_flags(target_ulong start, target_ulong end, int flags)
2391 target_ulong addr, len;
2393 /* This function should never be called with addresses outside the
2394 guest address space. If this assert fires, it probably indicates
2395 a missing call to h2g_valid. */
2396 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2397 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2398 #endif
2399 assert(start < end);
2401 start = start & TARGET_PAGE_MASK;
2402 end = TARGET_PAGE_ALIGN(end);
2404 if (flags & PAGE_WRITE) {
2405 flags |= PAGE_WRITE_ORG;
2408 for (addr = start, len = end - start;
2409 len != 0;
2410 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2411 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2413 /* If the write protection bit is set, then we invalidate
2414 the code inside. */
2415 if (!(p->flags & PAGE_WRITE) &&
2416 (flags & PAGE_WRITE) &&
2417 p->first_tb) {
2418 tb_invalidate_phys_page(addr, 0, NULL);
2420 p->flags = flags;
2424 int page_check_range(target_ulong start, target_ulong len, int flags)
2426 PageDesc *p;
2427 target_ulong end;
2428 target_ulong addr;
2430 /* This function should never be called with addresses outside the
2431 guest address space. If this assert fires, it probably indicates
2432 a missing call to h2g_valid. */
2433 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2434 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2435 #endif
2437 if (len == 0) {
2438 return 0;
2440 if (start + len - 1 < start) {
2441 /* We've wrapped around. */
2442 return -1;
2445 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2446 start = start & TARGET_PAGE_MASK;
2448 for (addr = start, len = end - start;
2449 len != 0;
2450 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2451 p = page_find(addr >> TARGET_PAGE_BITS);
2452 if( !p )
2453 return -1;
2454 if( !(p->flags & PAGE_VALID) )
2455 return -1;
2457 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2458 return -1;
2459 if (flags & PAGE_WRITE) {
2460 if (!(p->flags & PAGE_WRITE_ORG))
2461 return -1;
2462 /* unprotect the page if it was put read-only because it
2463 contains translated code */
2464 if (!(p->flags & PAGE_WRITE)) {
2465 if (!page_unprotect(addr, 0, NULL))
2466 return -1;
2468 return 0;
2471 return 0;
2474 /* called from signal handler: invalidate the code and unprotect the
2475 page. Return TRUE if the fault was successfully handled. */
2476 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2478 unsigned int prot;
2479 PageDesc *p;
2480 target_ulong host_start, host_end, addr;
2482 /* Technically this isn't safe inside a signal handler. However we
2483 know this only ever happens in a synchronous SEGV handler, so in
2484 practice it seems to be ok. */
2485 mmap_lock();
2487 p = page_find(address >> TARGET_PAGE_BITS);
2488 if (!p) {
2489 mmap_unlock();
2490 return 0;
2493 /* if the page was really writable, then we change its
2494 protection back to writable */
2495 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2496 host_start = address & qemu_host_page_mask;
2497 host_end = host_start + qemu_host_page_size;
2499 prot = 0;
2500 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2501 p = page_find(addr >> TARGET_PAGE_BITS);
2502 p->flags |= PAGE_WRITE;
2503 prot |= p->flags;
2505 /* and since the content will be modified, we must invalidate
2506 the corresponding translated code. */
2507 tb_invalidate_phys_page(addr, pc, puc);
2508 #ifdef DEBUG_TB_CHECK
2509 tb_invalidate_check(addr);
2510 #endif
2512 mprotect((void *)g2h(host_start), qemu_host_page_size,
2513 prot & PAGE_BITS);
2515 mmap_unlock();
2516 return 1;
2518 mmap_unlock();
2519 return 0;
2522 static inline void tlb_set_dirty(CPUState *env,
2523 unsigned long addr, target_ulong vaddr)
2526 #endif /* defined(CONFIG_USER_ONLY) */
2528 #if !defined(CONFIG_USER_ONLY)
2530 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2531 typedef struct subpage_t {
2532 target_phys_addr_t base;
2533 ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
2534 ram_addr_t region_offset[TARGET_PAGE_SIZE];
2535 } subpage_t;
2537 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2538 ram_addr_t memory, ram_addr_t region_offset);
2539 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2540 ram_addr_t orig_memory,
2541 ram_addr_t region_offset);
2542 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2543 need_subpage) \
2544 do { \
2545 if (addr > start_addr) \
2546 start_addr2 = 0; \
2547 else { \
2548 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2549 if (start_addr2 > 0) \
2550 need_subpage = 1; \
2553 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2554 end_addr2 = TARGET_PAGE_SIZE - 1; \
2555 else { \
2556 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2557 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2558 need_subpage = 1; \
2560 } while (0)
2562 /* register physical memory.
2563 For RAM, 'size' must be a multiple of the target page size.
2564 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2565 io memory page. The address used when calling the IO function is
2566 the offset from the start of the region, plus region_offset. Both
2567 start_addr and region_offset are rounded down to a page boundary
2568 before calculating this offset. This should not be a problem unless
2569 the low bits of start_addr and region_offset differ. */
2570 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2571 ram_addr_t size,
2572 ram_addr_t phys_offset,
2573 ram_addr_t region_offset)
2575 target_phys_addr_t addr, end_addr;
2576 PhysPageDesc *p;
2577 CPUState *env;
2578 ram_addr_t orig_size = size;
2579 subpage_t *subpage;
2581 cpu_notify_set_memory(start_addr, size, phys_offset);
2583 if (phys_offset == IO_MEM_UNASSIGNED) {
2584 region_offset = start_addr;
2586 region_offset &= TARGET_PAGE_MASK;
2587 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2588 end_addr = start_addr + (target_phys_addr_t)size;
2589 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2590 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2591 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2592 ram_addr_t orig_memory = p->phys_offset;
2593 target_phys_addr_t start_addr2, end_addr2;
2594 int need_subpage = 0;
2596 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2597 need_subpage);
2598 if (need_subpage) {
2599 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2600 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2601 &p->phys_offset, orig_memory,
2602 p->region_offset);
2603 } else {
2604 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2605 >> IO_MEM_SHIFT];
2607 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2608 region_offset);
2609 p->region_offset = 0;
2610 } else {
2611 p->phys_offset = phys_offset;
2612 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2613 (phys_offset & IO_MEM_ROMD))
2614 phys_offset += TARGET_PAGE_SIZE;
2616 } else {
2617 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2618 p->phys_offset = phys_offset;
2619 p->region_offset = region_offset;
2620 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2621 (phys_offset & IO_MEM_ROMD)) {
2622 phys_offset += TARGET_PAGE_SIZE;
2623 } else {
2624 target_phys_addr_t start_addr2, end_addr2;
2625 int need_subpage = 0;
2627 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2628 end_addr2, need_subpage);
2630 if (need_subpage) {
2631 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2632 &p->phys_offset, IO_MEM_UNASSIGNED,
2633 addr & TARGET_PAGE_MASK);
2634 subpage_register(subpage, start_addr2, end_addr2,
2635 phys_offset, region_offset);
2636 p->region_offset = 0;
2640 region_offset += TARGET_PAGE_SIZE;
2643 /* since each CPU stores ram addresses in its TLB cache, we must
2644 reset the modified entries */
2645 /* XXX: slow ! */
2646 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2647 tlb_flush(env, 1);
2651 /* XXX: temporary until new memory mapping API */
2652 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2654 PhysPageDesc *p;
2656 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2657 if (!p)
2658 return IO_MEM_UNASSIGNED;
2659 return p->phys_offset;
2662 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2664 if (kvm_enabled())
2665 kvm_coalesce_mmio_region(addr, size);
2668 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2670 if (kvm_enabled())
2671 kvm_uncoalesce_mmio_region(addr, size);
2674 void qemu_flush_coalesced_mmio_buffer(void)
2676 if (kvm_enabled())
2677 kvm_flush_coalesced_mmio_buffer();
2680 #if defined(__linux__) && !defined(TARGET_S390X)
2682 #include <sys/vfs.h>
2684 #define HUGETLBFS_MAGIC 0x958458f6
2686 static long gethugepagesize(const char *path)
2688 struct statfs fs;
2689 int ret;
2691 do {
2692 ret = statfs(path, &fs);
2693 } while (ret != 0 && errno == EINTR);
2695 if (ret != 0) {
2696 perror(path);
2697 return 0;
2700 if (fs.f_type != HUGETLBFS_MAGIC)
2701 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2703 return fs.f_bsize;
2706 static void *file_ram_alloc(ram_addr_t memory, const char *path)
2708 char *filename;
2709 void *area;
2710 int fd;
2711 #ifdef MAP_POPULATE
2712 int flags;
2713 #endif
2714 unsigned long hpagesize;
2716 hpagesize = gethugepagesize(path);
2717 if (!hpagesize) {
2718 return NULL;
2721 if (memory < hpagesize) {
2722 return NULL;
2725 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2726 fprintf(stderr, "host lacks kvm mmu notifiers, -mem-path unsupported\n");
2727 return NULL;
2730 if (asprintf(&filename, "%s/qemu_back_mem.XXXXXX", path) == -1) {
2731 return NULL;
2734 fd = mkstemp(filename);
2735 if (fd < 0) {
2736 perror("unable to create backing store for hugepages");
2737 free(filename);
2738 return NULL;
2740 unlink(filename);
2741 free(filename);
2743 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2746 * ftruncate is not supported by hugetlbfs in older
2747 * hosts, so don't bother bailing out on errors.
2748 * If anything goes wrong with it under other filesystems,
2749 * mmap will fail.
2751 if (ftruncate(fd, memory))
2752 perror("ftruncate");
2754 #ifdef MAP_POPULATE
2755 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2756 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2757 * to sidestep this quirk.
2759 flags = mem_prealloc ? MAP_POPULATE | MAP_SHARED : MAP_PRIVATE;
2760 area = mmap(0, memory, PROT_READ | PROT_WRITE, flags, fd, 0);
2761 #else
2762 area = mmap(0, memory, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0);
2763 #endif
2764 if (area == MAP_FAILED) {
2765 perror("file_ram_alloc: can't mmap RAM pages");
2766 close(fd);
2767 return (NULL);
2769 return area;
2771 #endif
2773 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2775 RAMBlock *new_block;
2777 size = TARGET_PAGE_ALIGN(size);
2778 new_block = qemu_malloc(sizeof(*new_block));
2780 if (mem_path) {
2781 #if defined (__linux__) && !defined(TARGET_S390X)
2782 new_block->host = file_ram_alloc(size, mem_path);
2783 if (!new_block->host) {
2784 new_block->host = qemu_vmalloc(size);
2785 #ifdef MADV_MERGEABLE
2786 madvise(new_block->host, size, MADV_MERGEABLE);
2787 #endif
2789 #else
2790 fprintf(stderr, "-mem-path option unsupported\n");
2791 exit(1);
2792 #endif
2793 } else {
2794 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2795 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2796 new_block->host = mmap((void*)0x1000000, size,
2797 PROT_EXEC|PROT_READ|PROT_WRITE,
2798 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2799 #else
2800 new_block->host = qemu_vmalloc(size);
2801 #endif
2802 #ifdef MADV_MERGEABLE
2803 madvise(new_block->host, size, MADV_MERGEABLE);
2804 #endif
2806 new_block->offset = last_ram_offset;
2807 new_block->length = size;
2809 new_block->next = ram_blocks;
2810 ram_blocks = new_block;
2812 phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2813 (last_ram_offset + size) >> TARGET_PAGE_BITS);
2814 memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2815 0xff, size >> TARGET_PAGE_BITS);
2817 last_ram_offset += size;
2819 if (kvm_enabled())
2820 kvm_setup_guest_memory(new_block->host, size);
2822 return new_block->offset;
2825 void qemu_ram_free(ram_addr_t addr)
2827 /* TODO: implement this. */
2830 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2831 With the exception of the softmmu code in this file, this should
2832 only be used for local memory (e.g. video ram) that the device owns,
2833 and knows it isn't going to access beyond the end of the block.
2835 It should not be used for general purpose DMA.
2836 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2838 void *qemu_get_ram_ptr(ram_addr_t addr)
2840 RAMBlock *prev;
2841 RAMBlock **prevp;
2842 RAMBlock *block;
2844 prev = NULL;
2845 prevp = &ram_blocks;
2846 block = ram_blocks;
2847 while (block && (block->offset > addr
2848 || block->offset + block->length <= addr)) {
2849 if (prev)
2850 prevp = &prev->next;
2851 prev = block;
2852 block = block->next;
2854 if (!block) {
2855 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2856 abort();
2858 /* Move this entry to to start of the list. */
2859 if (prev) {
2860 prev->next = block->next;
2861 block->next = *prevp;
2862 *prevp = block;
2864 return block->host + (addr - block->offset);
2867 /* Some of the softmmu routines need to translate from a host pointer
2868 (typically a TLB entry) back to a ram offset. */
2869 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2871 RAMBlock *block;
2872 uint8_t *host = ptr;
2874 block = ram_blocks;
2875 while (block && (block->host > host
2876 || block->host + block->length <= host)) {
2877 block = block->next;
2879 if (!block) {
2880 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2881 abort();
2883 return block->offset + (host - block->host);
2886 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2888 #ifdef DEBUG_UNASSIGNED
2889 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2890 #endif
2891 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2892 do_unassigned_access(addr, 0, 0, 0, 1);
2893 #endif
2894 return 0;
2897 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2899 #ifdef DEBUG_UNASSIGNED
2900 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2901 #endif
2902 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2903 do_unassigned_access(addr, 0, 0, 0, 2);
2904 #endif
2905 return 0;
2908 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2910 #ifdef DEBUG_UNASSIGNED
2911 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2912 #endif
2913 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2914 do_unassigned_access(addr, 0, 0, 0, 4);
2915 #endif
2916 return 0;
2919 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2921 #ifdef DEBUG_UNASSIGNED
2922 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2923 #endif
2924 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2925 do_unassigned_access(addr, 1, 0, 0, 1);
2926 #endif
2929 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2931 #ifdef DEBUG_UNASSIGNED
2932 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2933 #endif
2934 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2935 do_unassigned_access(addr, 1, 0, 0, 2);
2936 #endif
2939 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2941 #ifdef DEBUG_UNASSIGNED
2942 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2943 #endif
2944 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2945 do_unassigned_access(addr, 1, 0, 0, 4);
2946 #endif
2949 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
2950 unassigned_mem_readb,
2951 unassigned_mem_readw,
2952 unassigned_mem_readl,
2955 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
2956 unassigned_mem_writeb,
2957 unassigned_mem_writew,
2958 unassigned_mem_writel,
2961 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2962 uint32_t val)
2964 int dirty_flags;
2965 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
2966 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2967 #if !defined(CONFIG_USER_ONLY)
2968 tb_invalidate_phys_page_fast(ram_addr, 1);
2969 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
2970 #endif
2972 stb_p(qemu_get_ram_ptr(ram_addr), val);
2973 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2974 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
2975 /* we remove the notdirty callback only if the code has been
2976 flushed */
2977 if (dirty_flags == 0xff)
2978 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2981 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2982 uint32_t val)
2984 int dirty_flags;
2985 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
2986 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2987 #if !defined(CONFIG_USER_ONLY)
2988 tb_invalidate_phys_page_fast(ram_addr, 2);
2989 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
2990 #endif
2992 stw_p(qemu_get_ram_ptr(ram_addr), val);
2993 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2994 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
2995 /* we remove the notdirty callback only if the code has been
2996 flushed */
2997 if (dirty_flags == 0xff)
2998 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3001 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
3002 uint32_t val)
3004 int dirty_flags;
3005 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3006 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
3007 #if !defined(CONFIG_USER_ONLY)
3008 tb_invalidate_phys_page_fast(ram_addr, 4);
3009 dirty_flags = cpu_physical_memory_get_dirty_flags(ram_addr);
3010 #endif
3012 stl_p(qemu_get_ram_ptr(ram_addr), val);
3013 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
3014 cpu_physical_memory_set_dirty_flags(ram_addr, dirty_flags);
3015 /* we remove the notdirty callback only if the code has been
3016 flushed */
3017 if (dirty_flags == 0xff)
3018 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
3021 static CPUReadMemoryFunc * const error_mem_read[3] = {
3022 NULL, /* never used */
3023 NULL, /* never used */
3024 NULL, /* never used */
3027 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
3028 notdirty_mem_writeb,
3029 notdirty_mem_writew,
3030 notdirty_mem_writel,
3033 /* Generate a debug exception if a watchpoint has been hit. */
3034 static void check_watchpoint(int offset, int len_mask, int flags)
3036 CPUState *env = cpu_single_env;
3037 target_ulong pc, cs_base;
3038 TranslationBlock *tb;
3039 target_ulong vaddr;
3040 CPUWatchpoint *wp;
3041 int cpu_flags;
3043 if (env->watchpoint_hit) {
3044 /* We re-entered the check after replacing the TB. Now raise
3045 * the debug interrupt so that is will trigger after the
3046 * current instruction. */
3047 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
3048 return;
3050 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
3051 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
3052 if ((vaddr == (wp->vaddr & len_mask) ||
3053 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
3054 wp->flags |= BP_WATCHPOINT_HIT;
3055 if (!env->watchpoint_hit) {
3056 env->watchpoint_hit = wp;
3057 tb = tb_find_pc(env->mem_io_pc);
3058 if (!tb) {
3059 cpu_abort(env, "check_watchpoint: could not find TB for "
3060 "pc=%p", (void *)env->mem_io_pc);
3062 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
3063 tb_phys_invalidate(tb, -1);
3064 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
3065 env->exception_index = EXCP_DEBUG;
3066 } else {
3067 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
3068 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
3070 cpu_resume_from_signal(env, NULL);
3072 } else {
3073 wp->flags &= ~BP_WATCHPOINT_HIT;
3078 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
3079 so these check for a hit then pass through to the normal out-of-line
3080 phys routines. */
3081 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
3083 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
3084 return ldub_phys(addr);
3087 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
3089 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
3090 return lduw_phys(addr);
3093 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
3095 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
3096 return ldl_phys(addr);
3099 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
3100 uint32_t val)
3102 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
3103 stb_phys(addr, val);
3106 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
3107 uint32_t val)
3109 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
3110 stw_phys(addr, val);
3113 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
3114 uint32_t val)
3116 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
3117 stl_phys(addr, val);
3120 static CPUReadMemoryFunc * const watch_mem_read[3] = {
3121 watch_mem_readb,
3122 watch_mem_readw,
3123 watch_mem_readl,
3126 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
3127 watch_mem_writeb,
3128 watch_mem_writew,
3129 watch_mem_writel,
3132 static inline uint32_t subpage_readlen (subpage_t *mmio,
3133 target_phys_addr_t addr,
3134 unsigned int len)
3136 unsigned int idx = SUBPAGE_IDX(addr);
3137 #if defined(DEBUG_SUBPAGE)
3138 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
3139 mmio, len, addr, idx);
3140 #endif
3142 addr += mmio->region_offset[idx];
3143 idx = mmio->sub_io_index[idx];
3144 return io_mem_read[idx][len](io_mem_opaque[idx], addr);
3147 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
3148 uint32_t value, unsigned int len)
3150 unsigned int idx = SUBPAGE_IDX(addr);
3151 #if defined(DEBUG_SUBPAGE)
3152 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n",
3153 __func__, mmio, len, addr, idx, value);
3154 #endif
3156 addr += mmio->region_offset[idx];
3157 idx = mmio->sub_io_index[idx];
3158 io_mem_write[idx][len](io_mem_opaque[idx], addr, value);
3161 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
3163 return subpage_readlen(opaque, addr, 0);
3166 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
3167 uint32_t value)
3169 subpage_writelen(opaque, addr, value, 0);
3172 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
3174 return subpage_readlen(opaque, addr, 1);
3177 static void subpage_writew (void *opaque, target_phys_addr_t addr,
3178 uint32_t value)
3180 subpage_writelen(opaque, addr, value, 1);
3183 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3185 return subpage_readlen(opaque, addr, 2);
3188 static void subpage_writel (void *opaque, target_phys_addr_t addr,
3189 uint32_t value)
3191 subpage_writelen(opaque, addr, value, 2);
3194 static CPUReadMemoryFunc * const subpage_read[] = {
3195 &subpage_readb,
3196 &subpage_readw,
3197 &subpage_readl,
3200 static CPUWriteMemoryFunc * const subpage_write[] = {
3201 &subpage_writeb,
3202 &subpage_writew,
3203 &subpage_writel,
3206 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3207 ram_addr_t memory, ram_addr_t region_offset)
3209 int idx, eidx;
3211 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3212 return -1;
3213 idx = SUBPAGE_IDX(start);
3214 eidx = SUBPAGE_IDX(end);
3215 #if defined(DEBUG_SUBPAGE)
3216 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3217 mmio, start, end, idx, eidx, memory);
3218 #endif
3219 memory = (memory >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3220 for (; idx <= eidx; idx++) {
3221 mmio->sub_io_index[idx] = memory;
3222 mmio->region_offset[idx] = region_offset;
3225 return 0;
3228 static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3229 ram_addr_t orig_memory,
3230 ram_addr_t region_offset)
3232 subpage_t *mmio;
3233 int subpage_memory;
3235 mmio = qemu_mallocz(sizeof(subpage_t));
3237 mmio->base = base;
3238 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3239 #if defined(DEBUG_SUBPAGE)
3240 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3241 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3242 #endif
3243 *phys = subpage_memory | IO_MEM_SUBPAGE;
3244 subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
3246 return mmio;
3249 static int get_free_io_mem_idx(void)
3251 int i;
3253 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3254 if (!io_mem_used[i]) {
3255 io_mem_used[i] = 1;
3256 return i;
3258 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3259 return -1;
3262 /* mem_read and mem_write are arrays of functions containing the
3263 function to access byte (index 0), word (index 1) and dword (index
3264 2). Functions can be omitted with a NULL function pointer.
3265 If io_index is non zero, the corresponding io zone is
3266 modified. If it is zero, a new io zone is allocated. The return
3267 value can be used with cpu_register_physical_memory(). (-1) is
3268 returned if error. */
3269 static int cpu_register_io_memory_fixed(int io_index,
3270 CPUReadMemoryFunc * const *mem_read,
3271 CPUWriteMemoryFunc * const *mem_write,
3272 void *opaque)
3274 int i;
3276 if (io_index <= 0) {
3277 io_index = get_free_io_mem_idx();
3278 if (io_index == -1)
3279 return io_index;
3280 } else {
3281 io_index >>= IO_MEM_SHIFT;
3282 if (io_index >= IO_MEM_NB_ENTRIES)
3283 return -1;
3286 for (i = 0; i < 3; ++i) {
3287 io_mem_read[io_index][i]
3288 = (mem_read[i] ? mem_read[i] : unassigned_mem_read[i]);
3290 for (i = 0; i < 3; ++i) {
3291 io_mem_write[io_index][i]
3292 = (mem_write[i] ? mem_write[i] : unassigned_mem_write[i]);
3294 io_mem_opaque[io_index] = opaque;
3296 return (io_index << IO_MEM_SHIFT);
3299 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3300 CPUWriteMemoryFunc * const *mem_write,
3301 void *opaque)
3303 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3306 void cpu_unregister_io_memory(int io_table_address)
3308 int i;
3309 int io_index = io_table_address >> IO_MEM_SHIFT;
3311 for (i=0;i < 3; i++) {
3312 io_mem_read[io_index][i] = unassigned_mem_read[i];
3313 io_mem_write[io_index][i] = unassigned_mem_write[i];
3315 io_mem_opaque[io_index] = NULL;
3316 io_mem_used[io_index] = 0;
3319 static void io_mem_init(void)
3321 int i;
3323 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3324 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3325 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3326 for (i=0; i<5; i++)
3327 io_mem_used[i] = 1;
3329 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3330 watch_mem_write, NULL);
3333 #endif /* !defined(CONFIG_USER_ONLY) */
3335 /* physical memory access (slow version, mainly for debug) */
3336 #if defined(CONFIG_USER_ONLY)
3337 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3338 uint8_t *buf, int len, int is_write)
3340 int l, flags;
3341 target_ulong page;
3342 void * p;
3344 while (len > 0) {
3345 page = addr & TARGET_PAGE_MASK;
3346 l = (page + TARGET_PAGE_SIZE) - addr;
3347 if (l > len)
3348 l = len;
3349 flags = page_get_flags(page);
3350 if (!(flags & PAGE_VALID))
3351 return -1;
3352 if (is_write) {
3353 if (!(flags & PAGE_WRITE))
3354 return -1;
3355 /* XXX: this code should not depend on lock_user */
3356 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3357 return -1;
3358 memcpy(p, buf, l);
3359 unlock_user(p, addr, l);
3360 } else {
3361 if (!(flags & PAGE_READ))
3362 return -1;
3363 /* XXX: this code should not depend on lock_user */
3364 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3365 return -1;
3366 memcpy(buf, p, l);
3367 unlock_user(p, addr, 0);
3369 len -= l;
3370 buf += l;
3371 addr += l;
3373 return 0;
3376 #else
3377 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3378 int len, int is_write)
3380 int l, io_index;
3381 uint8_t *ptr;
3382 uint32_t val;
3383 target_phys_addr_t page;
3384 unsigned long pd;
3385 PhysPageDesc *p;
3387 while (len > 0) {
3388 page = addr & TARGET_PAGE_MASK;
3389 l = (page + TARGET_PAGE_SIZE) - addr;
3390 if (l > len)
3391 l = len;
3392 p = phys_page_find(page >> TARGET_PAGE_BITS);
3393 if (!p) {
3394 pd = IO_MEM_UNASSIGNED;
3395 } else {
3396 pd = p->phys_offset;
3399 if (is_write) {
3400 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3401 target_phys_addr_t addr1 = addr;
3402 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3403 if (p)
3404 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3405 /* XXX: could force cpu_single_env to NULL to avoid
3406 potential bugs */
3407 if (l >= 4 && ((addr1 & 3) == 0)) {
3408 /* 32 bit write access */
3409 val = ldl_p(buf);
3410 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3411 l = 4;
3412 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3413 /* 16 bit write access */
3414 val = lduw_p(buf);
3415 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3416 l = 2;
3417 } else {
3418 /* 8 bit write access */
3419 val = ldub_p(buf);
3420 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3421 l = 1;
3423 } else {
3424 unsigned long addr1;
3425 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3426 /* RAM case */
3427 ptr = qemu_get_ram_ptr(addr1);
3428 memcpy(ptr, buf, l);
3429 if (!cpu_physical_memory_is_dirty(addr1)) {
3430 /* invalidate code */
3431 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3432 /* set dirty bit */
3433 cpu_physical_memory_set_dirty_flags(
3434 addr1, (0xff & ~CODE_DIRTY_FLAG));
3437 } else {
3438 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3439 !(pd & IO_MEM_ROMD)) {
3440 target_phys_addr_t addr1 = addr;
3441 /* I/O case */
3442 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3443 if (p)
3444 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3445 if (l >= 4 && ((addr1 & 3) == 0)) {
3446 /* 32 bit read access */
3447 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3448 stl_p(buf, val);
3449 l = 4;
3450 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3451 /* 16 bit read access */
3452 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3453 stw_p(buf, val);
3454 l = 2;
3455 } else {
3456 /* 8 bit read access */
3457 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3458 stb_p(buf, val);
3459 l = 1;
3461 } else {
3462 /* RAM case */
3463 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3464 (addr & ~TARGET_PAGE_MASK);
3465 memcpy(buf, ptr, l);
3468 len -= l;
3469 buf += l;
3470 addr += l;
3474 /* used for ROM loading : can write in RAM and ROM */
3475 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3476 const uint8_t *buf, int len)
3478 int l;
3479 uint8_t *ptr;
3480 target_phys_addr_t page;
3481 unsigned long pd;
3482 PhysPageDesc *p;
3484 while (len > 0) {
3485 page = addr & TARGET_PAGE_MASK;
3486 l = (page + TARGET_PAGE_SIZE) - addr;
3487 if (l > len)
3488 l = len;
3489 p = phys_page_find(page >> TARGET_PAGE_BITS);
3490 if (!p) {
3491 pd = IO_MEM_UNASSIGNED;
3492 } else {
3493 pd = p->phys_offset;
3496 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3497 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3498 !(pd & IO_MEM_ROMD)) {
3499 /* do nothing */
3500 } else {
3501 unsigned long addr1;
3502 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3503 /* ROM/RAM case */
3504 ptr = qemu_get_ram_ptr(addr1);
3505 memcpy(ptr, buf, l);
3507 len -= l;
3508 buf += l;
3509 addr += l;
3513 typedef struct {
3514 void *buffer;
3515 target_phys_addr_t addr;
3516 target_phys_addr_t len;
3517 } BounceBuffer;
3519 static BounceBuffer bounce;
3521 typedef struct MapClient {
3522 void *opaque;
3523 void (*callback)(void *opaque);
3524 QLIST_ENTRY(MapClient) link;
3525 } MapClient;
3527 static QLIST_HEAD(map_client_list, MapClient) map_client_list
3528 = QLIST_HEAD_INITIALIZER(map_client_list);
3530 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3532 MapClient *client = qemu_malloc(sizeof(*client));
3534 client->opaque = opaque;
3535 client->callback = callback;
3536 QLIST_INSERT_HEAD(&map_client_list, client, link);
3537 return client;
3540 void cpu_unregister_map_client(void *_client)
3542 MapClient *client = (MapClient *)_client;
3544 QLIST_REMOVE(client, link);
3545 qemu_free(client);
3548 static void cpu_notify_map_clients(void)
3550 MapClient *client;
3552 while (!QLIST_EMPTY(&map_client_list)) {
3553 client = QLIST_FIRST(&map_client_list);
3554 client->callback(client->opaque);
3555 cpu_unregister_map_client(client);
3559 /* Map a physical memory region into a host virtual address.
3560 * May map a subset of the requested range, given by and returned in *plen.
3561 * May return NULL if resources needed to perform the mapping are exhausted.
3562 * Use only for reads OR writes - not for read-modify-write operations.
3563 * Use cpu_register_map_client() to know when retrying the map operation is
3564 * likely to succeed.
3566 void *cpu_physical_memory_map(target_phys_addr_t addr,
3567 target_phys_addr_t *plen,
3568 int is_write)
3570 target_phys_addr_t len = *plen;
3571 target_phys_addr_t done = 0;
3572 int l;
3573 uint8_t *ret = NULL;
3574 uint8_t *ptr;
3575 target_phys_addr_t page;
3576 unsigned long pd;
3577 PhysPageDesc *p;
3578 unsigned long addr1;
3580 while (len > 0) {
3581 page = addr & TARGET_PAGE_MASK;
3582 l = (page + TARGET_PAGE_SIZE) - addr;
3583 if (l > len)
3584 l = len;
3585 p = phys_page_find(page >> TARGET_PAGE_BITS);
3586 if (!p) {
3587 pd = IO_MEM_UNASSIGNED;
3588 } else {
3589 pd = p->phys_offset;
3592 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3593 if (done || bounce.buffer) {
3594 break;
3596 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3597 bounce.addr = addr;
3598 bounce.len = l;
3599 if (!is_write) {
3600 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3602 ptr = bounce.buffer;
3603 } else {
3604 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3605 ptr = qemu_get_ram_ptr(addr1);
3607 if (!done) {
3608 ret = ptr;
3609 } else if (ret + done != ptr) {
3610 break;
3613 len -= l;
3614 addr += l;
3615 done += l;
3617 *plen = done;
3618 return ret;
3621 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3622 * Will also mark the memory as dirty if is_write == 1. access_len gives
3623 * the amount of memory that was actually read or written by the caller.
3625 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3626 int is_write, target_phys_addr_t access_len)
3628 if (buffer != bounce.buffer) {
3629 if (is_write) {
3630 ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3631 while (access_len) {
3632 unsigned l;
3633 l = TARGET_PAGE_SIZE;
3634 if (l > access_len)
3635 l = access_len;
3636 if (!cpu_physical_memory_is_dirty(addr1)) {
3637 /* invalidate code */
3638 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3639 /* set dirty bit */
3640 cpu_physical_memory_set_dirty_flags(
3641 addr1, (0xff & ~CODE_DIRTY_FLAG));
3643 addr1 += l;
3644 access_len -= l;
3647 return;
3649 if (is_write) {
3650 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3652 qemu_vfree(bounce.buffer);
3653 bounce.buffer = NULL;
3654 cpu_notify_map_clients();
3657 /* warning: addr must be aligned */
3658 uint32_t ldl_phys(target_phys_addr_t addr)
3660 int io_index;
3661 uint8_t *ptr;
3662 uint32_t val;
3663 unsigned long pd;
3664 PhysPageDesc *p;
3666 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3667 if (!p) {
3668 pd = IO_MEM_UNASSIGNED;
3669 } else {
3670 pd = p->phys_offset;
3673 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3674 !(pd & IO_MEM_ROMD)) {
3675 /* I/O case */
3676 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3677 if (p)
3678 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3679 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3680 } else {
3681 /* RAM case */
3682 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3683 (addr & ~TARGET_PAGE_MASK);
3684 val = ldl_p(ptr);
3686 return val;
3689 /* warning: addr must be aligned */
3690 uint64_t ldq_phys(target_phys_addr_t addr)
3692 int io_index;
3693 uint8_t *ptr;
3694 uint64_t val;
3695 unsigned long pd;
3696 PhysPageDesc *p;
3698 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3699 if (!p) {
3700 pd = IO_MEM_UNASSIGNED;
3701 } else {
3702 pd = p->phys_offset;
3705 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3706 !(pd & IO_MEM_ROMD)) {
3707 /* I/O case */
3708 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3709 if (p)
3710 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3711 #ifdef TARGET_WORDS_BIGENDIAN
3712 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3713 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3714 #else
3715 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3716 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3717 #endif
3718 } else {
3719 /* RAM case */
3720 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3721 (addr & ~TARGET_PAGE_MASK);
3722 val = ldq_p(ptr);
3724 return val;
3727 /* XXX: optimize */
3728 uint32_t ldub_phys(target_phys_addr_t addr)
3730 uint8_t val;
3731 cpu_physical_memory_read(addr, &val, 1);
3732 return val;
3735 /* warning: addr must be aligned */
3736 uint32_t lduw_phys(target_phys_addr_t addr)
3738 int io_index;
3739 uint8_t *ptr;
3740 uint64_t val;
3741 unsigned long pd;
3742 PhysPageDesc *p;
3744 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3745 if (!p) {
3746 pd = IO_MEM_UNASSIGNED;
3747 } else {
3748 pd = p->phys_offset;
3751 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3752 !(pd & IO_MEM_ROMD)) {
3753 /* I/O case */
3754 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3755 if (p)
3756 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3757 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
3758 } else {
3759 /* RAM case */
3760 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3761 (addr & ~TARGET_PAGE_MASK);
3762 val = lduw_p(ptr);
3764 return val;
3767 /* warning: addr must be aligned. The ram page is not masked as dirty
3768 and the code inside is not invalidated. It is useful if the dirty
3769 bits are used to track modified PTEs */
3770 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3772 int io_index;
3773 uint8_t *ptr;
3774 unsigned long pd;
3775 PhysPageDesc *p;
3777 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3778 if (!p) {
3779 pd = IO_MEM_UNASSIGNED;
3780 } else {
3781 pd = p->phys_offset;
3784 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3785 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3786 if (p)
3787 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3788 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3789 } else {
3790 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3791 ptr = qemu_get_ram_ptr(addr1);
3792 stl_p(ptr, val);
3794 if (unlikely(in_migration)) {
3795 if (!cpu_physical_memory_is_dirty(addr1)) {
3796 /* invalidate code */
3797 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3798 /* set dirty bit */
3799 cpu_physical_memory_set_dirty_flags(
3800 addr1, (0xff & ~CODE_DIRTY_FLAG));
3806 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3808 int io_index;
3809 uint8_t *ptr;
3810 unsigned long pd;
3811 PhysPageDesc *p;
3813 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3814 if (!p) {
3815 pd = IO_MEM_UNASSIGNED;
3816 } else {
3817 pd = p->phys_offset;
3820 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3821 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3822 if (p)
3823 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3824 #ifdef TARGET_WORDS_BIGENDIAN
3825 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3826 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3827 #else
3828 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3829 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3830 #endif
3831 } else {
3832 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3833 (addr & ~TARGET_PAGE_MASK);
3834 stq_p(ptr, val);
3838 /* warning: addr must be aligned */
3839 void stl_phys(target_phys_addr_t addr, uint32_t val)
3841 int io_index;
3842 uint8_t *ptr;
3843 unsigned long pd;
3844 PhysPageDesc *p;
3846 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3847 if (!p) {
3848 pd = IO_MEM_UNASSIGNED;
3849 } else {
3850 pd = p->phys_offset;
3853 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3854 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3855 if (p)
3856 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3857 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3858 } else {
3859 unsigned long addr1;
3860 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3861 /* RAM case */
3862 ptr = qemu_get_ram_ptr(addr1);
3863 stl_p(ptr, val);
3864 if (!cpu_physical_memory_is_dirty(addr1)) {
3865 /* invalidate code */
3866 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3867 /* set dirty bit */
3868 cpu_physical_memory_set_dirty_flags(addr1,
3869 (0xff & ~CODE_DIRTY_FLAG));
3874 /* XXX: optimize */
3875 void stb_phys(target_phys_addr_t addr, uint32_t val)
3877 uint8_t v = val;
3878 cpu_physical_memory_write(addr, &v, 1);
3881 /* warning: addr must be aligned */
3882 void stw_phys(target_phys_addr_t addr, uint32_t val)
3884 int io_index;
3885 uint8_t *ptr;
3886 unsigned long pd;
3887 PhysPageDesc *p;
3889 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3890 if (!p) {
3891 pd = IO_MEM_UNASSIGNED;
3892 } else {
3893 pd = p->phys_offset;
3896 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3897 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3898 if (p)
3899 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3900 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
3901 } else {
3902 unsigned long addr1;
3903 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3904 /* RAM case */
3905 ptr = qemu_get_ram_ptr(addr1);
3906 stw_p(ptr, val);
3907 if (!cpu_physical_memory_is_dirty(addr1)) {
3908 /* invalidate code */
3909 tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
3910 /* set dirty bit */
3911 cpu_physical_memory_set_dirty_flags(addr1,
3912 (0xff & ~CODE_DIRTY_FLAG));
3917 /* XXX: optimize */
3918 void stq_phys(target_phys_addr_t addr, uint64_t val)
3920 val = tswap64(val);
3921 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3924 /* virtual memory access for debug (includes writing to ROM) */
3925 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3926 uint8_t *buf, int len, int is_write)
3928 int l;
3929 target_phys_addr_t phys_addr;
3930 target_ulong page;
3932 while (len > 0) {
3933 page = addr & TARGET_PAGE_MASK;
3934 phys_addr = cpu_get_phys_page_debug(env, page);
3935 /* if no physical page mapped, return an error */
3936 if (phys_addr == -1)
3937 return -1;
3938 l = (page + TARGET_PAGE_SIZE) - addr;
3939 if (l > len)
3940 l = len;
3941 phys_addr += (addr & ~TARGET_PAGE_MASK);
3942 if (is_write)
3943 cpu_physical_memory_write_rom(phys_addr, buf, l);
3944 else
3945 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3946 len -= l;
3947 buf += l;
3948 addr += l;
3950 return 0;
3952 #endif
3954 /* in deterministic execution mode, instructions doing device I/Os
3955 must be at the end of the TB */
3956 void cpu_io_recompile(CPUState *env, void *retaddr)
3958 TranslationBlock *tb;
3959 uint32_t n, cflags;
3960 target_ulong pc, cs_base;
3961 uint64_t flags;
3963 tb = tb_find_pc((unsigned long)retaddr);
3964 if (!tb) {
3965 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3966 retaddr);
3968 n = env->icount_decr.u16.low + tb->icount;
3969 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3970 /* Calculate how many instructions had been executed before the fault
3971 occurred. */
3972 n = n - env->icount_decr.u16.low;
3973 /* Generate a new TB ending on the I/O insn. */
3974 n++;
3975 /* On MIPS and SH, delay slot instructions can only be restarted if
3976 they were already the first instruction in the TB. If this is not
3977 the first instruction in a TB then re-execute the preceding
3978 branch. */
3979 #if defined(TARGET_MIPS)
3980 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3981 env->active_tc.PC -= 4;
3982 env->icount_decr.u16.low++;
3983 env->hflags &= ~MIPS_HFLAG_BMASK;
3985 #elif defined(TARGET_SH4)
3986 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3987 && n > 1) {
3988 env->pc -= 2;
3989 env->icount_decr.u16.low++;
3990 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3992 #endif
3993 /* This should never happen. */
3994 if (n > CF_COUNT_MASK)
3995 cpu_abort(env, "TB too big during recompile");
3997 cflags = n | CF_LAST_IO;
3998 pc = tb->pc;
3999 cs_base = tb->cs_base;
4000 flags = tb->flags;
4001 tb_phys_invalidate(tb, -1);
4002 /* FIXME: In theory this could raise an exception. In practice
4003 we have already translated the block once so it's probably ok. */
4004 tb_gen_code(env, pc, cs_base, flags, cflags);
4005 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
4006 the first in the TB) then we end up generating a whole new TB and
4007 repeating the fault, which is horribly inefficient.
4008 Better would be to execute just this insn uncached, or generate a
4009 second new TB. */
4010 cpu_resume_from_signal(env, NULL);
4013 #if !defined(CONFIG_USER_ONLY)
4015 void dump_exec_info(FILE *f,
4016 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
4018 int i, target_code_size, max_target_code_size;
4019 int direct_jmp_count, direct_jmp2_count, cross_page;
4020 TranslationBlock *tb;
4022 target_code_size = 0;
4023 max_target_code_size = 0;
4024 cross_page = 0;
4025 direct_jmp_count = 0;
4026 direct_jmp2_count = 0;
4027 for(i = 0; i < nb_tbs; i++) {
4028 tb = &tbs[i];
4029 target_code_size += tb->size;
4030 if (tb->size > max_target_code_size)
4031 max_target_code_size = tb->size;
4032 if (tb->page_addr[1] != -1)
4033 cross_page++;
4034 if (tb->tb_next_offset[0] != 0xffff) {
4035 direct_jmp_count++;
4036 if (tb->tb_next_offset[1] != 0xffff) {
4037 direct_jmp2_count++;
4041 /* XXX: avoid using doubles ? */
4042 cpu_fprintf(f, "Translation buffer state:\n");
4043 cpu_fprintf(f, "gen code size %ld/%ld\n",
4044 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
4045 cpu_fprintf(f, "TB count %d/%d\n",
4046 nb_tbs, code_gen_max_blocks);
4047 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
4048 nb_tbs ? target_code_size / nb_tbs : 0,
4049 max_target_code_size);
4050 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
4051 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
4052 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
4053 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
4054 cross_page,
4055 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
4056 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
4057 direct_jmp_count,
4058 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
4059 direct_jmp2_count,
4060 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
4061 cpu_fprintf(f, "\nStatistics:\n");
4062 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
4063 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
4064 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
4065 tcg_dump_info(f, cpu_fprintf);
4068 #define MMUSUFFIX _cmmu
4069 #define GETPC() NULL
4070 #define env cpu_single_env
4071 #define SOFTMMU_CODE_ACCESS
4073 #define SHIFT 0
4074 #include "softmmu_template.h"
4076 #define SHIFT 1
4077 #include "softmmu_template.h"
4079 #define SHIFT 2
4080 #include "softmmu_template.h"
4082 #define SHIFT 3
4083 #include "softmmu_template.h"
4085 #undef env
4087 #endif