eepro100: Keep includes sorted
[qemu.git] / exec.c
blob6a3c912b7f0a2867e6db4de0b26667232f1ed61b
1 /*
2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <stdarg.h>
29 #include <string.h>
30 #include <errno.h>
31 #include <unistd.h>
32 #include <inttypes.h>
34 #include "cpu.h"
35 #include "exec-all.h"
36 #include "qemu-common.h"
37 #include "tcg.h"
38 #include "hw/hw.h"
39 #include "osdep.h"
40 #include "kvm.h"
41 #if defined(CONFIG_USER_ONLY)
42 #include <qemu.h>
43 #include <signal.h>
44 #endif
46 //#define DEBUG_TB_INVALIDATE
47 //#define DEBUG_FLUSH
48 //#define DEBUG_TLB
49 //#define DEBUG_UNASSIGNED
51 /* make various TB consistency checks */
52 //#define DEBUG_TB_CHECK
53 //#define DEBUG_TLB_CHECK
55 //#define DEBUG_IOPORT
56 //#define DEBUG_SUBPAGE
58 #if !defined(CONFIG_USER_ONLY)
59 /* TB consistency checks only implemented for usermode emulation. */
60 #undef DEBUG_TB_CHECK
61 #endif
63 #define SMC_BITMAP_USE_THRESHOLD 10
65 #if defined(TARGET_SPARC64)
66 #define TARGET_PHYS_ADDR_SPACE_BITS 41
67 #elif defined(TARGET_SPARC)
68 #define TARGET_PHYS_ADDR_SPACE_BITS 36
69 #elif defined(TARGET_ALPHA)
70 #define TARGET_PHYS_ADDR_SPACE_BITS 42
71 #define TARGET_VIRT_ADDR_SPACE_BITS 42
72 #elif defined(TARGET_PPC64)
73 #define TARGET_PHYS_ADDR_SPACE_BITS 42
74 #elif defined(TARGET_X86_64)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 42
76 #elif defined(TARGET_I386)
77 #define TARGET_PHYS_ADDR_SPACE_BITS 36
78 #else
79 #define TARGET_PHYS_ADDR_SPACE_BITS 32
80 #endif
82 static TranslationBlock *tbs;
83 int code_gen_max_blocks;
84 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
85 static int nb_tbs;
86 /* any access to the tbs or the page table must use this lock */
87 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
89 #if defined(__arm__) || defined(__sparc_v9__)
90 /* The prologue must be reachable with a direct jump. ARM and Sparc64
91 have limited branch ranges (possibly also PPC) so place it in a
92 section close to code segment. */
93 #define code_gen_section \
94 __attribute__((__section__(".gen_code"))) \
95 __attribute__((aligned (32)))
96 #elif defined(_WIN32)
97 /* Maximum alignment for Win32 is 16. */
98 #define code_gen_section \
99 __attribute__((aligned (16)))
100 #else
101 #define code_gen_section \
102 __attribute__((aligned (32)))
103 #endif
105 uint8_t code_gen_prologue[1024] code_gen_section;
106 static uint8_t *code_gen_buffer;
107 static unsigned long code_gen_buffer_size;
108 /* threshold to flush the translated code buffer */
109 static unsigned long code_gen_buffer_max_size;
110 uint8_t *code_gen_ptr;
112 #if !defined(CONFIG_USER_ONLY)
113 int phys_ram_fd;
114 uint8_t *phys_ram_dirty;
115 static int in_migration;
117 typedef struct RAMBlock {
118 uint8_t *host;
119 ram_addr_t offset;
120 ram_addr_t length;
121 struct RAMBlock *next;
122 } RAMBlock;
124 static RAMBlock *ram_blocks;
125 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
126 then we can no longer assume contiguous ram offsets, and external uses
127 of this variable will break. */
128 ram_addr_t last_ram_offset;
129 #endif
131 CPUState *first_cpu;
132 /* current CPU in the current thread. It is only valid inside
133 cpu_exec() */
134 CPUState *cpu_single_env;
135 /* 0 = Do not count executed instructions.
136 1 = Precise instruction counting.
137 2 = Adaptive rate instruction counting. */
138 int use_icount = 0;
139 /* Current instruction counter. While executing translated code this may
140 include some instructions that have not yet been executed. */
141 int64_t qemu_icount;
143 typedef struct PageDesc {
144 /* list of TBs intersecting this ram page */
145 TranslationBlock *first_tb;
146 /* in order to optimize self modifying code, we count the number
147 of lookups we do to a given page to use a bitmap */
148 unsigned int code_write_count;
149 uint8_t *code_bitmap;
150 #if defined(CONFIG_USER_ONLY)
151 unsigned long flags;
152 #endif
153 } PageDesc;
155 typedef struct PhysPageDesc {
156 /* offset in host memory of the page + io_index in the low bits */
157 ram_addr_t phys_offset;
158 ram_addr_t region_offset;
159 } PhysPageDesc;
161 #define L2_BITS 10
162 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
163 /* XXX: this is a temporary hack for alpha target.
164 * In the future, this is to be replaced by a multi-level table
165 * to actually be able to handle the complete 64 bits address space.
167 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
168 #else
169 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
170 #endif
172 #define L1_SIZE (1 << L1_BITS)
173 #define L2_SIZE (1 << L2_BITS)
175 unsigned long qemu_real_host_page_size;
176 unsigned long qemu_host_page_bits;
177 unsigned long qemu_host_page_size;
178 unsigned long qemu_host_page_mask;
180 /* XXX: for system emulation, it could just be an array */
181 static PageDesc *l1_map[L1_SIZE];
183 #if !defined(CONFIG_USER_ONLY)
184 static PhysPageDesc **l1_phys_map;
186 static void io_mem_init(void);
188 /* io memory support */
189 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
190 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
191 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
192 static char io_mem_used[IO_MEM_NB_ENTRIES];
193 static int io_mem_watch;
194 #endif
196 /* log support */
197 #ifdef WIN32
198 static const char *logfilename = "qemu.log";
199 #else
200 static const char *logfilename = "/tmp/qemu.log";
201 #endif
202 FILE *logfile;
203 int loglevel;
204 static int log_append = 0;
206 /* statistics */
207 static int tlb_flush_count;
208 static int tb_flush_count;
209 static int tb_phys_invalidate_count;
211 #ifdef _WIN32
212 static void map_exec(void *addr, long size)
214 DWORD old_protect;
215 VirtualProtect(addr, size,
216 PAGE_EXECUTE_READWRITE, &old_protect);
219 #else
220 static void map_exec(void *addr, long size)
222 unsigned long start, end, page_size;
224 page_size = getpagesize();
225 start = (unsigned long)addr;
226 start &= ~(page_size - 1);
228 end = (unsigned long)addr + size;
229 end += page_size - 1;
230 end &= ~(page_size - 1);
232 mprotect((void *)start, end - start,
233 PROT_READ | PROT_WRITE | PROT_EXEC);
235 #endif
237 static void page_init(void)
239 /* NOTE: we can always suppose that qemu_host_page_size >=
240 TARGET_PAGE_SIZE */
241 #ifdef _WIN32
243 SYSTEM_INFO system_info;
245 GetSystemInfo(&system_info);
246 qemu_real_host_page_size = system_info.dwPageSize;
248 #else
249 qemu_real_host_page_size = getpagesize();
250 #endif
251 if (qemu_host_page_size == 0)
252 qemu_host_page_size = qemu_real_host_page_size;
253 if (qemu_host_page_size < TARGET_PAGE_SIZE)
254 qemu_host_page_size = TARGET_PAGE_SIZE;
255 qemu_host_page_bits = 0;
256 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
257 qemu_host_page_bits++;
258 qemu_host_page_mask = ~(qemu_host_page_size - 1);
259 #if !defined(CONFIG_USER_ONLY)
260 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
261 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
262 #endif
264 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
266 long long startaddr, endaddr;
267 FILE *f;
268 int n;
270 mmap_lock();
271 last_brk = (unsigned long)sbrk(0);
272 f = fopen("/proc/self/maps", "r");
273 if (f) {
274 do {
275 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
276 if (n == 2) {
277 startaddr = MIN(startaddr,
278 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
279 endaddr = MIN(endaddr,
280 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
281 page_set_flags(startaddr & TARGET_PAGE_MASK,
282 TARGET_PAGE_ALIGN(endaddr),
283 PAGE_RESERVED);
285 } while (!feof(f));
286 fclose(f);
288 mmap_unlock();
290 #endif
293 static inline PageDesc **page_l1_map(target_ulong index)
295 #if TARGET_LONG_BITS > 32
296 /* Host memory outside guest VM. For 32-bit targets we have already
297 excluded high addresses. */
298 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
299 return NULL;
300 #endif
301 return &l1_map[index >> L2_BITS];
304 static inline PageDesc *page_find_alloc(target_ulong index)
306 PageDesc **lp, *p;
307 lp = page_l1_map(index);
308 if (!lp)
309 return NULL;
311 p = *lp;
312 if (!p) {
313 /* allocate if not found */
314 #if defined(CONFIG_USER_ONLY)
315 size_t len = sizeof(PageDesc) * L2_SIZE;
316 /* Don't use qemu_malloc because it may recurse. */
317 p = mmap(NULL, len, PROT_READ | PROT_WRITE,
318 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
319 *lp = p;
320 if (h2g_valid(p)) {
321 unsigned long addr = h2g(p);
322 page_set_flags(addr & TARGET_PAGE_MASK,
323 TARGET_PAGE_ALIGN(addr + len),
324 PAGE_RESERVED);
326 #else
327 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
328 *lp = p;
329 #endif
331 return p + (index & (L2_SIZE - 1));
334 static inline PageDesc *page_find(target_ulong index)
336 PageDesc **lp, *p;
337 lp = page_l1_map(index);
338 if (!lp)
339 return NULL;
341 p = *lp;
342 if (!p) {
343 return NULL;
345 return p + (index & (L2_SIZE - 1));
348 #if !defined(CONFIG_USER_ONLY)
349 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
351 void **lp, **p;
352 PhysPageDesc *pd;
354 p = (void **)l1_phys_map;
355 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
357 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
358 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
359 #endif
360 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
361 p = *lp;
362 if (!p) {
363 /* allocate if not found */
364 if (!alloc)
365 return NULL;
366 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
367 memset(p, 0, sizeof(void *) * L1_SIZE);
368 *lp = p;
370 #endif
371 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
372 pd = *lp;
373 if (!pd) {
374 int i;
375 /* allocate if not found */
376 if (!alloc)
377 return NULL;
378 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
379 *lp = pd;
380 for (i = 0; i < L2_SIZE; i++) {
381 pd[i].phys_offset = IO_MEM_UNASSIGNED;
382 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
385 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
388 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
390 return phys_page_find_alloc(index, 0);
393 static void tlb_protect_code(ram_addr_t ram_addr);
394 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
395 target_ulong vaddr);
396 #define mmap_lock() do { } while(0)
397 #define mmap_unlock() do { } while(0)
398 #endif
400 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
402 #if defined(CONFIG_USER_ONLY)
403 /* Currently it is not recommended to allocate big chunks of data in
404 user mode. It will change when a dedicated libc will be used */
405 #define USE_STATIC_CODE_GEN_BUFFER
406 #endif
408 #ifdef USE_STATIC_CODE_GEN_BUFFER
409 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
410 #endif
412 static void code_gen_alloc(unsigned long tb_size)
414 #ifdef USE_STATIC_CODE_GEN_BUFFER
415 code_gen_buffer = static_code_gen_buffer;
416 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
417 map_exec(code_gen_buffer, code_gen_buffer_size);
418 #else
419 code_gen_buffer_size = tb_size;
420 if (code_gen_buffer_size == 0) {
421 #if defined(CONFIG_USER_ONLY)
422 /* in user mode, phys_ram_size is not meaningful */
423 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
424 #else
425 /* XXX: needs adjustments */
426 code_gen_buffer_size = (unsigned long)(ram_size / 4);
427 #endif
429 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
430 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
431 /* The code gen buffer location may have constraints depending on
432 the host cpu and OS */
433 #if defined(__linux__)
435 int flags;
436 void *start = NULL;
438 flags = MAP_PRIVATE | MAP_ANONYMOUS;
439 #if defined(__x86_64__)
440 flags |= MAP_32BIT;
441 /* Cannot map more than that */
442 if (code_gen_buffer_size > (800 * 1024 * 1024))
443 code_gen_buffer_size = (800 * 1024 * 1024);
444 #elif defined(__sparc_v9__)
445 // Map the buffer below 2G, so we can use direct calls and branches
446 flags |= MAP_FIXED;
447 start = (void *) 0x60000000UL;
448 if (code_gen_buffer_size > (512 * 1024 * 1024))
449 code_gen_buffer_size = (512 * 1024 * 1024);
450 #elif defined(__arm__)
451 /* Map the buffer below 32M, so we can use direct calls and branches */
452 flags |= MAP_FIXED;
453 start = (void *) 0x01000000UL;
454 if (code_gen_buffer_size > 16 * 1024 * 1024)
455 code_gen_buffer_size = 16 * 1024 * 1024;
456 #endif
457 code_gen_buffer = mmap(start, code_gen_buffer_size,
458 PROT_WRITE | PROT_READ | PROT_EXEC,
459 flags, -1, 0);
460 if (code_gen_buffer == MAP_FAILED) {
461 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
462 exit(1);
465 #elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(__DragonFly__)
467 int flags;
468 void *addr = NULL;
469 flags = MAP_PRIVATE | MAP_ANONYMOUS;
470 #if defined(__x86_64__)
471 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
472 * 0x40000000 is free */
473 flags |= MAP_FIXED;
474 addr = (void *)0x40000000;
475 /* Cannot map more than that */
476 if (code_gen_buffer_size > (800 * 1024 * 1024))
477 code_gen_buffer_size = (800 * 1024 * 1024);
478 #endif
479 code_gen_buffer = mmap(addr, code_gen_buffer_size,
480 PROT_WRITE | PROT_READ | PROT_EXEC,
481 flags, -1, 0);
482 if (code_gen_buffer == MAP_FAILED) {
483 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
484 exit(1);
487 #else
488 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
489 map_exec(code_gen_buffer, code_gen_buffer_size);
490 #endif
491 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
492 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
493 code_gen_buffer_max_size = code_gen_buffer_size -
494 code_gen_max_block_size();
495 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
496 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
499 /* Must be called before using the QEMU cpus. 'tb_size' is the size
500 (in bytes) allocated to the translation buffer. Zero means default
501 size. */
502 void cpu_exec_init_all(unsigned long tb_size)
504 cpu_gen_init();
505 code_gen_alloc(tb_size);
506 code_gen_ptr = code_gen_buffer;
507 page_init();
508 #if !defined(CONFIG_USER_ONLY)
509 io_mem_init();
510 #endif
513 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
515 static void cpu_common_pre_save(void *opaque)
517 CPUState *env = opaque;
519 cpu_synchronize_state(env);
522 static int cpu_common_pre_load(void *opaque)
524 CPUState *env = opaque;
526 cpu_synchronize_state(env);
527 return 0;
530 static int cpu_common_post_load(void *opaque, int version_id)
532 CPUState *env = opaque;
534 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
535 version_id is increased. */
536 env->interrupt_request &= ~0x01;
537 tlb_flush(env, 1);
539 return 0;
542 static const VMStateDescription vmstate_cpu_common = {
543 .name = "cpu_common",
544 .version_id = 1,
545 .minimum_version_id = 1,
546 .minimum_version_id_old = 1,
547 .pre_save = cpu_common_pre_save,
548 .pre_load = cpu_common_pre_load,
549 .post_load = cpu_common_post_load,
550 .fields = (VMStateField []) {
551 VMSTATE_UINT32(halted, CPUState),
552 VMSTATE_UINT32(interrupt_request, CPUState),
553 VMSTATE_END_OF_LIST()
556 #endif
558 CPUState *qemu_get_cpu(int cpu)
560 CPUState *env = first_cpu;
562 while (env) {
563 if (env->cpu_index == cpu)
564 break;
565 env = env->next_cpu;
568 return env;
571 void cpu_exec_init(CPUState *env)
573 CPUState **penv;
574 int cpu_index;
576 #if defined(CONFIG_USER_ONLY)
577 cpu_list_lock();
578 #endif
579 env->next_cpu = NULL;
580 penv = &first_cpu;
581 cpu_index = 0;
582 while (*penv != NULL) {
583 penv = &(*penv)->next_cpu;
584 cpu_index++;
586 env->cpu_index = cpu_index;
587 env->numa_node = 0;
588 QTAILQ_INIT(&env->breakpoints);
589 QTAILQ_INIT(&env->watchpoints);
590 *penv = env;
591 #if defined(CONFIG_USER_ONLY)
592 cpu_list_unlock();
593 #endif
594 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
595 vmstate_register(cpu_index, &vmstate_cpu_common, env);
596 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
597 cpu_save, cpu_load, env);
598 #endif
601 static inline void invalidate_page_bitmap(PageDesc *p)
603 if (p->code_bitmap) {
604 qemu_free(p->code_bitmap);
605 p->code_bitmap = NULL;
607 p->code_write_count = 0;
610 /* set to NULL all the 'first_tb' fields in all PageDescs */
611 static void page_flush_tb(void)
613 int i, j;
614 PageDesc *p;
616 for(i = 0; i < L1_SIZE; i++) {
617 p = l1_map[i];
618 if (p) {
619 for(j = 0; j < L2_SIZE; j++) {
620 p->first_tb = NULL;
621 invalidate_page_bitmap(p);
622 p++;
628 /* flush all the translation blocks */
629 /* XXX: tb_flush is currently not thread safe */
630 void tb_flush(CPUState *env1)
632 CPUState *env;
633 #if defined(DEBUG_FLUSH)
634 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
635 (unsigned long)(code_gen_ptr - code_gen_buffer),
636 nb_tbs, nb_tbs > 0 ?
637 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
638 #endif
639 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
640 cpu_abort(env1, "Internal error: code buffer overflow\n");
642 nb_tbs = 0;
644 for(env = first_cpu; env != NULL; env = env->next_cpu) {
645 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
648 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
649 page_flush_tb();
651 code_gen_ptr = code_gen_buffer;
652 /* XXX: flush processor icache at this point if cache flush is
653 expensive */
654 tb_flush_count++;
657 #ifdef DEBUG_TB_CHECK
659 static void tb_invalidate_check(target_ulong address)
661 TranslationBlock *tb;
662 int i;
663 address &= TARGET_PAGE_MASK;
664 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
665 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
666 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
667 address >= tb->pc + tb->size)) {
668 printf("ERROR invalidate: address=" TARGET_FMT_lx
669 " PC=%08lx size=%04x\n",
670 address, (long)tb->pc, tb->size);
676 /* verify that all the pages have correct rights for code */
677 static void tb_page_check(void)
679 TranslationBlock *tb;
680 int i, flags1, flags2;
682 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
683 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
684 flags1 = page_get_flags(tb->pc);
685 flags2 = page_get_flags(tb->pc + tb->size - 1);
686 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
687 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
688 (long)tb->pc, tb->size, flags1, flags2);
694 #endif
696 /* invalidate one TB */
697 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
698 int next_offset)
700 TranslationBlock *tb1;
701 for(;;) {
702 tb1 = *ptb;
703 if (tb1 == tb) {
704 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
705 break;
707 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
711 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
713 TranslationBlock *tb1;
714 unsigned int n1;
716 for(;;) {
717 tb1 = *ptb;
718 n1 = (long)tb1 & 3;
719 tb1 = (TranslationBlock *)((long)tb1 & ~3);
720 if (tb1 == tb) {
721 *ptb = tb1->page_next[n1];
722 break;
724 ptb = &tb1->page_next[n1];
728 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
730 TranslationBlock *tb1, **ptb;
731 unsigned int n1;
733 ptb = &tb->jmp_next[n];
734 tb1 = *ptb;
735 if (tb1) {
736 /* find tb(n) in circular list */
737 for(;;) {
738 tb1 = *ptb;
739 n1 = (long)tb1 & 3;
740 tb1 = (TranslationBlock *)((long)tb1 & ~3);
741 if (n1 == n && tb1 == tb)
742 break;
743 if (n1 == 2) {
744 ptb = &tb1->jmp_first;
745 } else {
746 ptb = &tb1->jmp_next[n1];
749 /* now we can suppress tb(n) from the list */
750 *ptb = tb->jmp_next[n];
752 tb->jmp_next[n] = NULL;
756 /* reset the jump entry 'n' of a TB so that it is not chained to
757 another TB */
758 static inline void tb_reset_jump(TranslationBlock *tb, int n)
760 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
763 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
765 CPUState *env;
766 PageDesc *p;
767 unsigned int h, n1;
768 target_phys_addr_t phys_pc;
769 TranslationBlock *tb1, *tb2;
771 /* remove the TB from the hash list */
772 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
773 h = tb_phys_hash_func(phys_pc);
774 tb_remove(&tb_phys_hash[h], tb,
775 offsetof(TranslationBlock, phys_hash_next));
777 /* remove the TB from the page list */
778 if (tb->page_addr[0] != page_addr) {
779 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
780 tb_page_remove(&p->first_tb, tb);
781 invalidate_page_bitmap(p);
783 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
784 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
785 tb_page_remove(&p->first_tb, tb);
786 invalidate_page_bitmap(p);
789 tb_invalidated_flag = 1;
791 /* remove the TB from the hash list */
792 h = tb_jmp_cache_hash_func(tb->pc);
793 for(env = first_cpu; env != NULL; env = env->next_cpu) {
794 if (env->tb_jmp_cache[h] == tb)
795 env->tb_jmp_cache[h] = NULL;
798 /* suppress this TB from the two jump lists */
799 tb_jmp_remove(tb, 0);
800 tb_jmp_remove(tb, 1);
802 /* suppress any remaining jumps to this TB */
803 tb1 = tb->jmp_first;
804 for(;;) {
805 n1 = (long)tb1 & 3;
806 if (n1 == 2)
807 break;
808 tb1 = (TranslationBlock *)((long)tb1 & ~3);
809 tb2 = tb1->jmp_next[n1];
810 tb_reset_jump(tb1, n1);
811 tb1->jmp_next[n1] = NULL;
812 tb1 = tb2;
814 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
816 tb_phys_invalidate_count++;
819 static inline void set_bits(uint8_t *tab, int start, int len)
821 int end, mask, end1;
823 end = start + len;
824 tab += start >> 3;
825 mask = 0xff << (start & 7);
826 if ((start & ~7) == (end & ~7)) {
827 if (start < end) {
828 mask &= ~(0xff << (end & 7));
829 *tab |= mask;
831 } else {
832 *tab++ |= mask;
833 start = (start + 8) & ~7;
834 end1 = end & ~7;
835 while (start < end1) {
836 *tab++ = 0xff;
837 start += 8;
839 if (start < end) {
840 mask = ~(0xff << (end & 7));
841 *tab |= mask;
846 static void build_page_bitmap(PageDesc *p)
848 int n, tb_start, tb_end;
849 TranslationBlock *tb;
851 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
853 tb = p->first_tb;
854 while (tb != NULL) {
855 n = (long)tb & 3;
856 tb = (TranslationBlock *)((long)tb & ~3);
857 /* NOTE: this is subtle as a TB may span two physical pages */
858 if (n == 0) {
859 /* NOTE: tb_end may be after the end of the page, but
860 it is not a problem */
861 tb_start = tb->pc & ~TARGET_PAGE_MASK;
862 tb_end = tb_start + tb->size;
863 if (tb_end > TARGET_PAGE_SIZE)
864 tb_end = TARGET_PAGE_SIZE;
865 } else {
866 tb_start = 0;
867 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
869 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
870 tb = tb->page_next[n];
874 TranslationBlock *tb_gen_code(CPUState *env,
875 target_ulong pc, target_ulong cs_base,
876 int flags, int cflags)
878 TranslationBlock *tb;
879 uint8_t *tc_ptr;
880 target_ulong phys_pc, phys_page2, virt_page2;
881 int code_gen_size;
883 phys_pc = get_phys_addr_code(env, pc);
884 tb = tb_alloc(pc);
885 if (!tb) {
886 /* flush must be done */
887 tb_flush(env);
888 /* cannot fail at this point */
889 tb = tb_alloc(pc);
890 /* Don't forget to invalidate previous TB info. */
891 tb_invalidated_flag = 1;
893 tc_ptr = code_gen_ptr;
894 tb->tc_ptr = tc_ptr;
895 tb->cs_base = cs_base;
896 tb->flags = flags;
897 tb->cflags = cflags;
898 cpu_gen_code(env, tb, &code_gen_size);
899 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
901 /* check next page if needed */
902 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
903 phys_page2 = -1;
904 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
905 phys_page2 = get_phys_addr_code(env, virt_page2);
907 tb_link_phys(tb, phys_pc, phys_page2);
908 return tb;
911 /* invalidate all TBs which intersect with the target physical page
912 starting in range [start;end[. NOTE: start and end must refer to
913 the same physical page. 'is_cpu_write_access' should be true if called
914 from a real cpu write access: the virtual CPU will exit the current
915 TB if code is modified inside this TB. */
916 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
917 int is_cpu_write_access)
919 TranslationBlock *tb, *tb_next, *saved_tb;
920 CPUState *env = cpu_single_env;
921 target_ulong tb_start, tb_end;
922 PageDesc *p;
923 int n;
924 #ifdef TARGET_HAS_PRECISE_SMC
925 int current_tb_not_found = is_cpu_write_access;
926 TranslationBlock *current_tb = NULL;
927 int current_tb_modified = 0;
928 target_ulong current_pc = 0;
929 target_ulong current_cs_base = 0;
930 int current_flags = 0;
931 #endif /* TARGET_HAS_PRECISE_SMC */
933 p = page_find(start >> TARGET_PAGE_BITS);
934 if (!p)
935 return;
936 if (!p->code_bitmap &&
937 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
938 is_cpu_write_access) {
939 /* build code bitmap */
940 build_page_bitmap(p);
943 /* we remove all the TBs in the range [start, end[ */
944 /* XXX: see if in some cases it could be faster to invalidate all the code */
945 tb = p->first_tb;
946 while (tb != NULL) {
947 n = (long)tb & 3;
948 tb = (TranslationBlock *)((long)tb & ~3);
949 tb_next = tb->page_next[n];
950 /* NOTE: this is subtle as a TB may span two physical pages */
951 if (n == 0) {
952 /* NOTE: tb_end may be after the end of the page, but
953 it is not a problem */
954 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
955 tb_end = tb_start + tb->size;
956 } else {
957 tb_start = tb->page_addr[1];
958 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
960 if (!(tb_end <= start || tb_start >= end)) {
961 #ifdef TARGET_HAS_PRECISE_SMC
962 if (current_tb_not_found) {
963 current_tb_not_found = 0;
964 current_tb = NULL;
965 if (env->mem_io_pc) {
966 /* now we have a real cpu fault */
967 current_tb = tb_find_pc(env->mem_io_pc);
970 if (current_tb == tb &&
971 (current_tb->cflags & CF_COUNT_MASK) != 1) {
972 /* If we are modifying the current TB, we must stop
973 its execution. We could be more precise by checking
974 that the modification is after the current PC, but it
975 would require a specialized function to partially
976 restore the CPU state */
978 current_tb_modified = 1;
979 cpu_restore_state(current_tb, env,
980 env->mem_io_pc, NULL);
981 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
982 &current_flags);
984 #endif /* TARGET_HAS_PRECISE_SMC */
985 /* we need to do that to handle the case where a signal
986 occurs while doing tb_phys_invalidate() */
987 saved_tb = NULL;
988 if (env) {
989 saved_tb = env->current_tb;
990 env->current_tb = NULL;
992 tb_phys_invalidate(tb, -1);
993 if (env) {
994 env->current_tb = saved_tb;
995 if (env->interrupt_request && env->current_tb)
996 cpu_interrupt(env, env->interrupt_request);
999 tb = tb_next;
1001 #if !defined(CONFIG_USER_ONLY)
1002 /* if no code remaining, no need to continue to use slow writes */
1003 if (!p->first_tb) {
1004 invalidate_page_bitmap(p);
1005 if (is_cpu_write_access) {
1006 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1009 #endif
1010 #ifdef TARGET_HAS_PRECISE_SMC
1011 if (current_tb_modified) {
1012 /* we generate a block containing just the instruction
1013 modifying the memory. It will ensure that it cannot modify
1014 itself */
1015 env->current_tb = NULL;
1016 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1017 cpu_resume_from_signal(env, NULL);
1019 #endif
1022 /* len must be <= 8 and start must be a multiple of len */
1023 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1025 PageDesc *p;
1026 int offset, b;
1027 #if 0
1028 if (1) {
1029 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1030 cpu_single_env->mem_io_vaddr, len,
1031 cpu_single_env->eip,
1032 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1034 #endif
1035 p = page_find(start >> TARGET_PAGE_BITS);
1036 if (!p)
1037 return;
1038 if (p->code_bitmap) {
1039 offset = start & ~TARGET_PAGE_MASK;
1040 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1041 if (b & ((1 << len) - 1))
1042 goto do_invalidate;
1043 } else {
1044 do_invalidate:
1045 tb_invalidate_phys_page_range(start, start + len, 1);
1049 #if !defined(CONFIG_SOFTMMU)
1050 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1051 unsigned long pc, void *puc)
1053 TranslationBlock *tb;
1054 PageDesc *p;
1055 int n;
1056 #ifdef TARGET_HAS_PRECISE_SMC
1057 TranslationBlock *current_tb = NULL;
1058 CPUState *env = cpu_single_env;
1059 int current_tb_modified = 0;
1060 target_ulong current_pc = 0;
1061 target_ulong current_cs_base = 0;
1062 int current_flags = 0;
1063 #endif
1065 addr &= TARGET_PAGE_MASK;
1066 p = page_find(addr >> TARGET_PAGE_BITS);
1067 if (!p)
1068 return;
1069 tb = p->first_tb;
1070 #ifdef TARGET_HAS_PRECISE_SMC
1071 if (tb && pc != 0) {
1072 current_tb = tb_find_pc(pc);
1074 #endif
1075 while (tb != NULL) {
1076 n = (long)tb & 3;
1077 tb = (TranslationBlock *)((long)tb & ~3);
1078 #ifdef TARGET_HAS_PRECISE_SMC
1079 if (current_tb == tb &&
1080 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1081 /* If we are modifying the current TB, we must stop
1082 its execution. We could be more precise by checking
1083 that the modification is after the current PC, but it
1084 would require a specialized function to partially
1085 restore the CPU state */
1087 current_tb_modified = 1;
1088 cpu_restore_state(current_tb, env, pc, puc);
1089 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1090 &current_flags);
1092 #endif /* TARGET_HAS_PRECISE_SMC */
1093 tb_phys_invalidate(tb, addr);
1094 tb = tb->page_next[n];
1096 p->first_tb = NULL;
1097 #ifdef TARGET_HAS_PRECISE_SMC
1098 if (current_tb_modified) {
1099 /* we generate a block containing just the instruction
1100 modifying the memory. It will ensure that it cannot modify
1101 itself */
1102 env->current_tb = NULL;
1103 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1104 cpu_resume_from_signal(env, puc);
1106 #endif
1108 #endif
1110 /* add the tb in the target page and protect it if necessary */
1111 static inline void tb_alloc_page(TranslationBlock *tb,
1112 unsigned int n, target_ulong page_addr)
1114 PageDesc *p;
1115 TranslationBlock *last_first_tb;
1117 tb->page_addr[n] = page_addr;
1118 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1119 tb->page_next[n] = p->first_tb;
1120 last_first_tb = p->first_tb;
1121 p->first_tb = (TranslationBlock *)((long)tb | n);
1122 invalidate_page_bitmap(p);
1124 #if defined(TARGET_HAS_SMC) || 1
1126 #if defined(CONFIG_USER_ONLY)
1127 if (p->flags & PAGE_WRITE) {
1128 target_ulong addr;
1129 PageDesc *p2;
1130 int prot;
1132 /* force the host page as non writable (writes will have a
1133 page fault + mprotect overhead) */
1134 page_addr &= qemu_host_page_mask;
1135 prot = 0;
1136 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1137 addr += TARGET_PAGE_SIZE) {
1139 p2 = page_find (addr >> TARGET_PAGE_BITS);
1140 if (!p2)
1141 continue;
1142 prot |= p2->flags;
1143 p2->flags &= ~PAGE_WRITE;
1144 page_get_flags(addr);
1146 mprotect(g2h(page_addr), qemu_host_page_size,
1147 (prot & PAGE_BITS) & ~PAGE_WRITE);
1148 #ifdef DEBUG_TB_INVALIDATE
1149 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1150 page_addr);
1151 #endif
1153 #else
1154 /* if some code is already present, then the pages are already
1155 protected. So we handle the case where only the first TB is
1156 allocated in a physical page */
1157 if (!last_first_tb) {
1158 tlb_protect_code(page_addr);
1160 #endif
1162 #endif /* TARGET_HAS_SMC */
1165 /* Allocate a new translation block. Flush the translation buffer if
1166 too many translation blocks or too much generated code. */
1167 TranslationBlock *tb_alloc(target_ulong pc)
1169 TranslationBlock *tb;
1171 if (nb_tbs >= code_gen_max_blocks ||
1172 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1173 return NULL;
1174 tb = &tbs[nb_tbs++];
1175 tb->pc = pc;
1176 tb->cflags = 0;
1177 return tb;
1180 void tb_free(TranslationBlock *tb)
1182 /* In practice this is mostly used for single use temporary TB
1183 Ignore the hard cases and just back up if this TB happens to
1184 be the last one generated. */
1185 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1186 code_gen_ptr = tb->tc_ptr;
1187 nb_tbs--;
1191 /* add a new TB and link it to the physical page tables. phys_page2 is
1192 (-1) to indicate that only one page contains the TB. */
1193 void tb_link_phys(TranslationBlock *tb,
1194 target_ulong phys_pc, target_ulong phys_page2)
1196 unsigned int h;
1197 TranslationBlock **ptb;
1199 /* Grab the mmap lock to stop another thread invalidating this TB
1200 before we are done. */
1201 mmap_lock();
1202 /* add in the physical hash table */
1203 h = tb_phys_hash_func(phys_pc);
1204 ptb = &tb_phys_hash[h];
1205 tb->phys_hash_next = *ptb;
1206 *ptb = tb;
1208 /* add in the page list */
1209 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1210 if (phys_page2 != -1)
1211 tb_alloc_page(tb, 1, phys_page2);
1212 else
1213 tb->page_addr[1] = -1;
1215 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1216 tb->jmp_next[0] = NULL;
1217 tb->jmp_next[1] = NULL;
1219 /* init original jump addresses */
1220 if (tb->tb_next_offset[0] != 0xffff)
1221 tb_reset_jump(tb, 0);
1222 if (tb->tb_next_offset[1] != 0xffff)
1223 tb_reset_jump(tb, 1);
1225 #ifdef DEBUG_TB_CHECK
1226 tb_page_check();
1227 #endif
1228 mmap_unlock();
1231 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1232 tb[1].tc_ptr. Return NULL if not found */
1233 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1235 int m_min, m_max, m;
1236 unsigned long v;
1237 TranslationBlock *tb;
1239 if (nb_tbs <= 0)
1240 return NULL;
1241 if (tc_ptr < (unsigned long)code_gen_buffer ||
1242 tc_ptr >= (unsigned long)code_gen_ptr)
1243 return NULL;
1244 /* binary search (cf Knuth) */
1245 m_min = 0;
1246 m_max = nb_tbs - 1;
1247 while (m_min <= m_max) {
1248 m = (m_min + m_max) >> 1;
1249 tb = &tbs[m];
1250 v = (unsigned long)tb->tc_ptr;
1251 if (v == tc_ptr)
1252 return tb;
1253 else if (tc_ptr < v) {
1254 m_max = m - 1;
1255 } else {
1256 m_min = m + 1;
1259 return &tbs[m_max];
1262 static void tb_reset_jump_recursive(TranslationBlock *tb);
1264 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1266 TranslationBlock *tb1, *tb_next, **ptb;
1267 unsigned int n1;
1269 tb1 = tb->jmp_next[n];
1270 if (tb1 != NULL) {
1271 /* find head of list */
1272 for(;;) {
1273 n1 = (long)tb1 & 3;
1274 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1275 if (n1 == 2)
1276 break;
1277 tb1 = tb1->jmp_next[n1];
1279 /* we are now sure now that tb jumps to tb1 */
1280 tb_next = tb1;
1282 /* remove tb from the jmp_first list */
1283 ptb = &tb_next->jmp_first;
1284 for(;;) {
1285 tb1 = *ptb;
1286 n1 = (long)tb1 & 3;
1287 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1288 if (n1 == n && tb1 == tb)
1289 break;
1290 ptb = &tb1->jmp_next[n1];
1292 *ptb = tb->jmp_next[n];
1293 tb->jmp_next[n] = NULL;
1295 /* suppress the jump to next tb in generated code */
1296 tb_reset_jump(tb, n);
1298 /* suppress jumps in the tb on which we could have jumped */
1299 tb_reset_jump_recursive(tb_next);
1303 static void tb_reset_jump_recursive(TranslationBlock *tb)
1305 tb_reset_jump_recursive2(tb, 0);
1306 tb_reset_jump_recursive2(tb, 1);
1309 #if defined(TARGET_HAS_ICE)
1310 #if defined(CONFIG_USER_ONLY)
1311 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1313 tb_invalidate_phys_page_range(pc, pc + 1, 0);
1315 #else
1316 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1318 target_phys_addr_t addr;
1319 target_ulong pd;
1320 ram_addr_t ram_addr;
1321 PhysPageDesc *p;
1323 addr = cpu_get_phys_page_debug(env, pc);
1324 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1325 if (!p) {
1326 pd = IO_MEM_UNASSIGNED;
1327 } else {
1328 pd = p->phys_offset;
1330 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1331 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1333 #endif
1334 #endif /* TARGET_HAS_ICE */
1336 #if defined(CONFIG_USER_ONLY)
1337 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1342 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1343 int flags, CPUWatchpoint **watchpoint)
1345 return -ENOSYS;
1347 #else
1348 /* Add a watchpoint. */
1349 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1350 int flags, CPUWatchpoint **watchpoint)
1352 target_ulong len_mask = ~(len - 1);
1353 CPUWatchpoint *wp;
1355 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1356 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1357 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1358 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1359 return -EINVAL;
1361 wp = qemu_malloc(sizeof(*wp));
1363 wp->vaddr = addr;
1364 wp->len_mask = len_mask;
1365 wp->flags = flags;
1367 /* keep all GDB-injected watchpoints in front */
1368 if (flags & BP_GDB)
1369 QTAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1370 else
1371 QTAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1373 tlb_flush_page(env, addr);
1375 if (watchpoint)
1376 *watchpoint = wp;
1377 return 0;
1380 /* Remove a specific watchpoint. */
1381 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1382 int flags)
1384 target_ulong len_mask = ~(len - 1);
1385 CPUWatchpoint *wp;
1387 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1388 if (addr == wp->vaddr && len_mask == wp->len_mask
1389 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1390 cpu_watchpoint_remove_by_ref(env, wp);
1391 return 0;
1394 return -ENOENT;
1397 /* Remove a specific watchpoint by reference. */
1398 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1400 QTAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1402 tlb_flush_page(env, watchpoint->vaddr);
1404 qemu_free(watchpoint);
1407 /* Remove all matching watchpoints. */
1408 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1410 CPUWatchpoint *wp, *next;
1412 QTAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1413 if (wp->flags & mask)
1414 cpu_watchpoint_remove_by_ref(env, wp);
1417 #endif
1419 /* Add a breakpoint. */
1420 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1421 CPUBreakpoint **breakpoint)
1423 #if defined(TARGET_HAS_ICE)
1424 CPUBreakpoint *bp;
1426 bp = qemu_malloc(sizeof(*bp));
1428 bp->pc = pc;
1429 bp->flags = flags;
1431 /* keep all GDB-injected breakpoints in front */
1432 if (flags & BP_GDB)
1433 QTAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1434 else
1435 QTAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1437 breakpoint_invalidate(env, pc);
1439 if (breakpoint)
1440 *breakpoint = bp;
1441 return 0;
1442 #else
1443 return -ENOSYS;
1444 #endif
1447 /* Remove a specific breakpoint. */
1448 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1450 #if defined(TARGET_HAS_ICE)
1451 CPUBreakpoint *bp;
1453 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1454 if (bp->pc == pc && bp->flags == flags) {
1455 cpu_breakpoint_remove_by_ref(env, bp);
1456 return 0;
1459 return -ENOENT;
1460 #else
1461 return -ENOSYS;
1462 #endif
1465 /* Remove a specific breakpoint by reference. */
1466 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1468 #if defined(TARGET_HAS_ICE)
1469 QTAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1471 breakpoint_invalidate(env, breakpoint->pc);
1473 qemu_free(breakpoint);
1474 #endif
1477 /* Remove all matching breakpoints. */
1478 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1480 #if defined(TARGET_HAS_ICE)
1481 CPUBreakpoint *bp, *next;
1483 QTAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1484 if (bp->flags & mask)
1485 cpu_breakpoint_remove_by_ref(env, bp);
1487 #endif
1490 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1491 CPU loop after each instruction */
1492 void cpu_single_step(CPUState *env, int enabled)
1494 #if defined(TARGET_HAS_ICE)
1495 if (env->singlestep_enabled != enabled) {
1496 env->singlestep_enabled = enabled;
1497 if (kvm_enabled())
1498 kvm_update_guest_debug(env, 0);
1499 else {
1500 /* must flush all the translated code to avoid inconsistencies */
1501 /* XXX: only flush what is necessary */
1502 tb_flush(env);
1505 #endif
1508 /* enable or disable low levels log */
1509 void cpu_set_log(int log_flags)
1511 loglevel = log_flags;
1512 if (loglevel && !logfile) {
1513 logfile = fopen(logfilename, log_append ? "a" : "w");
1514 if (!logfile) {
1515 perror(logfilename);
1516 _exit(1);
1518 #if !defined(CONFIG_SOFTMMU)
1519 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1521 static char logfile_buf[4096];
1522 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1524 #elif !defined(_WIN32)
1525 /* Win32 doesn't support line-buffering and requires size >= 2 */
1526 setvbuf(logfile, NULL, _IOLBF, 0);
1527 #endif
1528 log_append = 1;
1530 if (!loglevel && logfile) {
1531 fclose(logfile);
1532 logfile = NULL;
1536 void cpu_set_log_filename(const char *filename)
1538 logfilename = strdup(filename);
1539 if (logfile) {
1540 fclose(logfile);
1541 logfile = NULL;
1543 cpu_set_log(loglevel);
1546 static void cpu_unlink_tb(CPUState *env)
1548 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1549 problem and hope the cpu will stop of its own accord. For userspace
1550 emulation this often isn't actually as bad as it sounds. Often
1551 signals are used primarily to interrupt blocking syscalls. */
1552 TranslationBlock *tb;
1553 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1555 spin_lock(&interrupt_lock);
1556 tb = env->current_tb;
1557 /* if the cpu is currently executing code, we must unlink it and
1558 all the potentially executing TB */
1559 if (tb) {
1560 env->current_tb = NULL;
1561 tb_reset_jump_recursive(tb);
1563 spin_unlock(&interrupt_lock);
1566 /* mask must never be zero, except for A20 change call */
1567 void cpu_interrupt(CPUState *env, int mask)
1569 int old_mask;
1571 old_mask = env->interrupt_request;
1572 env->interrupt_request |= mask;
1574 #ifndef CONFIG_USER_ONLY
1576 * If called from iothread context, wake the target cpu in
1577 * case its halted.
1579 if (!qemu_cpu_self(env)) {
1580 qemu_cpu_kick(env);
1581 return;
1583 #endif
1585 if (use_icount) {
1586 env->icount_decr.u16.high = 0xffff;
1587 #ifndef CONFIG_USER_ONLY
1588 if (!can_do_io(env)
1589 && (mask & ~old_mask) != 0) {
1590 cpu_abort(env, "Raised interrupt while not in I/O function");
1592 #endif
1593 } else {
1594 cpu_unlink_tb(env);
1598 void cpu_reset_interrupt(CPUState *env, int mask)
1600 env->interrupt_request &= ~mask;
1603 void cpu_exit(CPUState *env)
1605 env->exit_request = 1;
1606 cpu_unlink_tb(env);
1609 const CPULogItem cpu_log_items[] = {
1610 { CPU_LOG_TB_OUT_ASM, "out_asm",
1611 "show generated host assembly code for each compiled TB" },
1612 { CPU_LOG_TB_IN_ASM, "in_asm",
1613 "show target assembly code for each compiled TB" },
1614 { CPU_LOG_TB_OP, "op",
1615 "show micro ops for each compiled TB" },
1616 { CPU_LOG_TB_OP_OPT, "op_opt",
1617 "show micro ops "
1618 #ifdef TARGET_I386
1619 "before eflags optimization and "
1620 #endif
1621 "after liveness analysis" },
1622 { CPU_LOG_INT, "int",
1623 "show interrupts/exceptions in short format" },
1624 { CPU_LOG_EXEC, "exec",
1625 "show trace before each executed TB (lots of logs)" },
1626 { CPU_LOG_TB_CPU, "cpu",
1627 "show CPU state before block translation" },
1628 #ifdef TARGET_I386
1629 { CPU_LOG_PCALL, "pcall",
1630 "show protected mode far calls/returns/exceptions" },
1631 { CPU_LOG_RESET, "cpu_reset",
1632 "show CPU state before CPU resets" },
1633 #endif
1634 #ifdef DEBUG_IOPORT
1635 { CPU_LOG_IOPORT, "ioport",
1636 "show all i/o ports accesses" },
1637 #endif
1638 { 0, NULL, NULL },
1641 #ifndef CONFIG_USER_ONLY
1642 static QLIST_HEAD(memory_client_list, CPUPhysMemoryClient) memory_client_list
1643 = QLIST_HEAD_INITIALIZER(memory_client_list);
1645 static void cpu_notify_set_memory(target_phys_addr_t start_addr,
1646 ram_addr_t size,
1647 ram_addr_t phys_offset)
1649 CPUPhysMemoryClient *client;
1650 QLIST_FOREACH(client, &memory_client_list, list) {
1651 client->set_memory(client, start_addr, size, phys_offset);
1655 static int cpu_notify_sync_dirty_bitmap(target_phys_addr_t start,
1656 target_phys_addr_t end)
1658 CPUPhysMemoryClient *client;
1659 QLIST_FOREACH(client, &memory_client_list, list) {
1660 int r = client->sync_dirty_bitmap(client, start, end);
1661 if (r < 0)
1662 return r;
1664 return 0;
1667 static int cpu_notify_migration_log(int enable)
1669 CPUPhysMemoryClient *client;
1670 QLIST_FOREACH(client, &memory_client_list, list) {
1671 int r = client->migration_log(client, enable);
1672 if (r < 0)
1673 return r;
1675 return 0;
1678 static void phys_page_for_each_in_l1_map(PhysPageDesc **phys_map,
1679 CPUPhysMemoryClient *client)
1681 PhysPageDesc *pd;
1682 int l1, l2;
1684 for (l1 = 0; l1 < L1_SIZE; ++l1) {
1685 pd = phys_map[l1];
1686 if (!pd) {
1687 continue;
1689 for (l2 = 0; l2 < L2_SIZE; ++l2) {
1690 if (pd[l2].phys_offset == IO_MEM_UNASSIGNED) {
1691 continue;
1693 client->set_memory(client, pd[l2].region_offset,
1694 TARGET_PAGE_SIZE, pd[l2].phys_offset);
1699 static void phys_page_for_each(CPUPhysMemoryClient *client)
1701 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
1703 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
1704 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
1705 #endif
1706 void **phys_map = (void **)l1_phys_map;
1707 int l1;
1708 if (!l1_phys_map) {
1709 return;
1711 for (l1 = 0; l1 < L1_SIZE; ++l1) {
1712 if (phys_map[l1]) {
1713 phys_page_for_each_in_l1_map(phys_map[l1], client);
1716 #else
1717 if (!l1_phys_map) {
1718 return;
1720 phys_page_for_each_in_l1_map(l1_phys_map, client);
1721 #endif
1724 void cpu_register_phys_memory_client(CPUPhysMemoryClient *client)
1726 QLIST_INSERT_HEAD(&memory_client_list, client, list);
1727 phys_page_for_each(client);
1730 void cpu_unregister_phys_memory_client(CPUPhysMemoryClient *client)
1732 QLIST_REMOVE(client, list);
1734 #endif
1736 static int cmp1(const char *s1, int n, const char *s2)
1738 if (strlen(s2) != n)
1739 return 0;
1740 return memcmp(s1, s2, n) == 0;
1743 /* takes a comma separated list of log masks. Return 0 if error. */
1744 int cpu_str_to_log_mask(const char *str)
1746 const CPULogItem *item;
1747 int mask;
1748 const char *p, *p1;
1750 p = str;
1751 mask = 0;
1752 for(;;) {
1753 p1 = strchr(p, ',');
1754 if (!p1)
1755 p1 = p + strlen(p);
1756 if(cmp1(p,p1-p,"all")) {
1757 for(item = cpu_log_items; item->mask != 0; item++) {
1758 mask |= item->mask;
1760 } else {
1761 for(item = cpu_log_items; item->mask != 0; item++) {
1762 if (cmp1(p, p1 - p, item->name))
1763 goto found;
1765 return 0;
1767 found:
1768 mask |= item->mask;
1769 if (*p1 != ',')
1770 break;
1771 p = p1 + 1;
1773 return mask;
1776 void cpu_abort(CPUState *env, const char *fmt, ...)
1778 va_list ap;
1779 va_list ap2;
1781 va_start(ap, fmt);
1782 va_copy(ap2, ap);
1783 fprintf(stderr, "qemu: fatal: ");
1784 vfprintf(stderr, fmt, ap);
1785 fprintf(stderr, "\n");
1786 #ifdef TARGET_I386
1787 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1788 #else
1789 cpu_dump_state(env, stderr, fprintf, 0);
1790 #endif
1791 if (qemu_log_enabled()) {
1792 qemu_log("qemu: fatal: ");
1793 qemu_log_vprintf(fmt, ap2);
1794 qemu_log("\n");
1795 #ifdef TARGET_I386
1796 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1797 #else
1798 log_cpu_state(env, 0);
1799 #endif
1800 qemu_log_flush();
1801 qemu_log_close();
1803 va_end(ap2);
1804 va_end(ap);
1805 #if defined(CONFIG_USER_ONLY)
1807 struct sigaction act;
1808 sigfillset(&act.sa_mask);
1809 act.sa_handler = SIG_DFL;
1810 sigaction(SIGABRT, &act, NULL);
1812 #endif
1813 abort();
1816 CPUState *cpu_copy(CPUState *env)
1818 CPUState *new_env = cpu_init(env->cpu_model_str);
1819 CPUState *next_cpu = new_env->next_cpu;
1820 int cpu_index = new_env->cpu_index;
1821 #if defined(TARGET_HAS_ICE)
1822 CPUBreakpoint *bp;
1823 CPUWatchpoint *wp;
1824 #endif
1826 memcpy(new_env, env, sizeof(CPUState));
1828 /* Preserve chaining and index. */
1829 new_env->next_cpu = next_cpu;
1830 new_env->cpu_index = cpu_index;
1832 /* Clone all break/watchpoints.
1833 Note: Once we support ptrace with hw-debug register access, make sure
1834 BP_CPU break/watchpoints are handled correctly on clone. */
1835 QTAILQ_INIT(&env->breakpoints);
1836 QTAILQ_INIT(&env->watchpoints);
1837 #if defined(TARGET_HAS_ICE)
1838 QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
1839 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1841 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
1842 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1843 wp->flags, NULL);
1845 #endif
1847 return new_env;
1850 #if !defined(CONFIG_USER_ONLY)
1852 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1854 unsigned int i;
1856 /* Discard jump cache entries for any tb which might potentially
1857 overlap the flushed page. */
1858 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1859 memset (&env->tb_jmp_cache[i], 0,
1860 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1862 i = tb_jmp_cache_hash_page(addr);
1863 memset (&env->tb_jmp_cache[i], 0,
1864 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1867 static CPUTLBEntry s_cputlb_empty_entry = {
1868 .addr_read = -1,
1869 .addr_write = -1,
1870 .addr_code = -1,
1871 .addend = -1,
1874 /* NOTE: if flush_global is true, also flush global entries (not
1875 implemented yet) */
1876 void tlb_flush(CPUState *env, int flush_global)
1878 int i;
1880 #if defined(DEBUG_TLB)
1881 printf("tlb_flush:\n");
1882 #endif
1883 /* must reset current TB so that interrupts cannot modify the
1884 links while we are modifying them */
1885 env->current_tb = NULL;
1887 for(i = 0; i < CPU_TLB_SIZE; i++) {
1888 int mmu_idx;
1889 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1890 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1894 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1896 tlb_flush_count++;
1899 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1901 if (addr == (tlb_entry->addr_read &
1902 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1903 addr == (tlb_entry->addr_write &
1904 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1905 addr == (tlb_entry->addr_code &
1906 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1907 *tlb_entry = s_cputlb_empty_entry;
1911 void tlb_flush_page(CPUState *env, target_ulong addr)
1913 int i;
1914 int mmu_idx;
1916 #if defined(DEBUG_TLB)
1917 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1918 #endif
1919 /* must reset current TB so that interrupts cannot modify the
1920 links while we are modifying them */
1921 env->current_tb = NULL;
1923 addr &= TARGET_PAGE_MASK;
1924 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1925 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1926 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1928 tlb_flush_jmp_cache(env, addr);
1931 /* update the TLBs so that writes to code in the virtual page 'addr'
1932 can be detected */
1933 static void tlb_protect_code(ram_addr_t ram_addr)
1935 cpu_physical_memory_reset_dirty(ram_addr,
1936 ram_addr + TARGET_PAGE_SIZE,
1937 CODE_DIRTY_FLAG);
1940 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1941 tested for self modifying code */
1942 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1943 target_ulong vaddr)
1945 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1948 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1949 unsigned long start, unsigned long length)
1951 unsigned long addr;
1952 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1953 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1954 if ((addr - start) < length) {
1955 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1960 /* Note: start and end must be within the same ram block. */
1961 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1962 int dirty_flags)
1964 CPUState *env;
1965 unsigned long length, start1;
1966 int i, mask, len;
1967 uint8_t *p;
1969 start &= TARGET_PAGE_MASK;
1970 end = TARGET_PAGE_ALIGN(end);
1972 length = end - start;
1973 if (length == 0)
1974 return;
1975 len = length >> TARGET_PAGE_BITS;
1976 mask = ~dirty_flags;
1977 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1978 for(i = 0; i < len; i++)
1979 p[i] &= mask;
1981 /* we modify the TLB cache so that the dirty bit will be set again
1982 when accessing the range */
1983 start1 = (unsigned long)qemu_get_ram_ptr(start);
1984 /* Chek that we don't span multiple blocks - this breaks the
1985 address comparisons below. */
1986 if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
1987 != (end - 1) - start) {
1988 abort();
1991 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1992 int mmu_idx;
1993 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1994 for(i = 0; i < CPU_TLB_SIZE; i++)
1995 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
1996 start1, length);
2001 int cpu_physical_memory_set_dirty_tracking(int enable)
2003 int ret = 0;
2004 in_migration = enable;
2005 ret = cpu_notify_migration_log(!!enable);
2006 return ret;
2009 int cpu_physical_memory_get_dirty_tracking(void)
2011 return in_migration;
2014 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
2015 target_phys_addr_t end_addr)
2017 int ret;
2019 ret = cpu_notify_sync_dirty_bitmap(start_addr, end_addr);
2020 return ret;
2023 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
2025 ram_addr_t ram_addr;
2026 void *p;
2028 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
2029 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
2030 + tlb_entry->addend);
2031 ram_addr = qemu_ram_addr_from_host(p);
2032 if (!cpu_physical_memory_is_dirty(ram_addr)) {
2033 tlb_entry->addr_write |= TLB_NOTDIRTY;
2038 /* update the TLB according to the current state of the dirty bits */
2039 void cpu_tlb_update_dirty(CPUState *env)
2041 int i;
2042 int mmu_idx;
2043 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
2044 for(i = 0; i < CPU_TLB_SIZE; i++)
2045 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
2049 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
2051 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
2052 tlb_entry->addr_write = vaddr;
2055 /* update the TLB corresponding to virtual page vaddr
2056 so that it is no longer dirty */
2057 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
2059 int i;
2060 int mmu_idx;
2062 vaddr &= TARGET_PAGE_MASK;
2063 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2064 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
2065 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
2068 /* add a new TLB entry. At most one entry for a given virtual address
2069 is permitted. Return 0 if OK or 2 if the page could not be mapped
2070 (can only happen in non SOFTMMU mode for I/O pages or pages
2071 conflicting with the host address space). */
2072 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2073 target_phys_addr_t paddr, int prot,
2074 int mmu_idx, int is_softmmu)
2076 PhysPageDesc *p;
2077 unsigned long pd;
2078 unsigned int index;
2079 target_ulong address;
2080 target_ulong code_address;
2081 target_phys_addr_t addend;
2082 int ret;
2083 CPUTLBEntry *te;
2084 CPUWatchpoint *wp;
2085 target_phys_addr_t iotlb;
2087 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2088 if (!p) {
2089 pd = IO_MEM_UNASSIGNED;
2090 } else {
2091 pd = p->phys_offset;
2093 #if defined(DEBUG_TLB)
2094 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2095 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2096 #endif
2098 ret = 0;
2099 address = vaddr;
2100 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2101 /* IO memory case (romd handled later) */
2102 address |= TLB_MMIO;
2104 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2105 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2106 /* Normal RAM. */
2107 iotlb = pd & TARGET_PAGE_MASK;
2108 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2109 iotlb |= IO_MEM_NOTDIRTY;
2110 else
2111 iotlb |= IO_MEM_ROM;
2112 } else {
2113 /* IO handlers are currently passed a physical address.
2114 It would be nice to pass an offset from the base address
2115 of that region. This would avoid having to special case RAM,
2116 and avoid full address decoding in every device.
2117 We can't use the high bits of pd for this because
2118 IO_MEM_ROMD uses these as a ram address. */
2119 iotlb = (pd & ~TARGET_PAGE_MASK);
2120 if (p) {
2121 iotlb += p->region_offset;
2122 } else {
2123 iotlb += paddr;
2127 code_address = address;
2128 /* Make accesses to pages with watchpoints go via the
2129 watchpoint trap routines. */
2130 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2131 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2132 iotlb = io_mem_watch + paddr;
2133 /* TODO: The memory case can be optimized by not trapping
2134 reads of pages with a write breakpoint. */
2135 address |= TLB_MMIO;
2139 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2140 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2141 te = &env->tlb_table[mmu_idx][index];
2142 te->addend = addend - vaddr;
2143 if (prot & PAGE_READ) {
2144 te->addr_read = address;
2145 } else {
2146 te->addr_read = -1;
2149 if (prot & PAGE_EXEC) {
2150 te->addr_code = code_address;
2151 } else {
2152 te->addr_code = -1;
2154 if (prot & PAGE_WRITE) {
2155 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2156 (pd & IO_MEM_ROMD)) {
2157 /* Write access calls the I/O callback. */
2158 te->addr_write = address | TLB_MMIO;
2159 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2160 !cpu_physical_memory_is_dirty(pd)) {
2161 te->addr_write = address | TLB_NOTDIRTY;
2162 } else {
2163 te->addr_write = address;
2165 } else {
2166 te->addr_write = -1;
2168 return ret;
2171 #else
2173 void tlb_flush(CPUState *env, int flush_global)
2177 void tlb_flush_page(CPUState *env, target_ulong addr)
2182 * Walks guest process memory "regions" one by one
2183 * and calls callback function 'fn' for each region.
2185 int walk_memory_regions(void *priv,
2186 int (*fn)(void *, unsigned long, unsigned long, unsigned long))
2188 unsigned long start, end;
2189 PageDesc *p = NULL;
2190 int i, j, prot, prot1;
2191 int rc = 0;
2193 start = end = -1;
2194 prot = 0;
2196 for (i = 0; i <= L1_SIZE; i++) {
2197 p = (i < L1_SIZE) ? l1_map[i] : NULL;
2198 for (j = 0; j < L2_SIZE; j++) {
2199 prot1 = (p == NULL) ? 0 : p[j].flags;
2201 * "region" is one continuous chunk of memory
2202 * that has same protection flags set.
2204 if (prot1 != prot) {
2205 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2206 if (start != -1) {
2207 rc = (*fn)(priv, start, end, prot);
2208 /* callback can stop iteration by returning != 0 */
2209 if (rc != 0)
2210 return (rc);
2212 if (prot1 != 0)
2213 start = end;
2214 else
2215 start = -1;
2216 prot = prot1;
2218 if (p == NULL)
2219 break;
2222 return (rc);
2225 static int dump_region(void *priv, unsigned long start,
2226 unsigned long end, unsigned long prot)
2228 FILE *f = (FILE *)priv;
2230 (void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2231 start, end, end - start,
2232 ((prot & PAGE_READ) ? 'r' : '-'),
2233 ((prot & PAGE_WRITE) ? 'w' : '-'),
2234 ((prot & PAGE_EXEC) ? 'x' : '-'));
2236 return (0);
2239 /* dump memory mappings */
2240 void page_dump(FILE *f)
2242 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2243 "start", "end", "size", "prot");
2244 walk_memory_regions(f, dump_region);
2247 int page_get_flags(target_ulong address)
2249 PageDesc *p;
2251 p = page_find(address >> TARGET_PAGE_BITS);
2252 if (!p)
2253 return 0;
2254 return p->flags;
2257 /* modify the flags of a page and invalidate the code if
2258 necessary. The flag PAGE_WRITE_ORG is positioned automatically
2259 depending on PAGE_WRITE */
2260 void page_set_flags(target_ulong start, target_ulong end, int flags)
2262 PageDesc *p;
2263 target_ulong addr;
2265 /* mmap_lock should already be held. */
2266 start = start & TARGET_PAGE_MASK;
2267 end = TARGET_PAGE_ALIGN(end);
2268 if (flags & PAGE_WRITE)
2269 flags |= PAGE_WRITE_ORG;
2270 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2271 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2272 /* We may be called for host regions that are outside guest
2273 address space. */
2274 if (!p)
2275 return;
2276 /* if the write protection is set, then we invalidate the code
2277 inside */
2278 if (!(p->flags & PAGE_WRITE) &&
2279 (flags & PAGE_WRITE) &&
2280 p->first_tb) {
2281 tb_invalidate_phys_page(addr, 0, NULL);
2283 p->flags = flags;
2287 int page_check_range(target_ulong start, target_ulong len, int flags)
2289 PageDesc *p;
2290 target_ulong end;
2291 target_ulong addr;
2293 if (start + len < start)
2294 /* we've wrapped around */
2295 return -1;
2297 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2298 start = start & TARGET_PAGE_MASK;
2300 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2301 p = page_find(addr >> TARGET_PAGE_BITS);
2302 if( !p )
2303 return -1;
2304 if( !(p->flags & PAGE_VALID) )
2305 return -1;
2307 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2308 return -1;
2309 if (flags & PAGE_WRITE) {
2310 if (!(p->flags & PAGE_WRITE_ORG))
2311 return -1;
2312 /* unprotect the page if it was put read-only because it
2313 contains translated code */
2314 if (!(p->flags & PAGE_WRITE)) {
2315 if (!page_unprotect(addr, 0, NULL))
2316 return -1;
2318 return 0;
2321 return 0;
2324 /* called from signal handler: invalidate the code and unprotect the
2325 page. Return TRUE if the fault was successfully handled. */
2326 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2328 unsigned int page_index, prot, pindex;
2329 PageDesc *p, *p1;
2330 target_ulong host_start, host_end, addr;
2332 /* Technically this isn't safe inside a signal handler. However we
2333 know this only ever happens in a synchronous SEGV handler, so in
2334 practice it seems to be ok. */
2335 mmap_lock();
2337 host_start = address & qemu_host_page_mask;
2338 page_index = host_start >> TARGET_PAGE_BITS;
2339 p1 = page_find(page_index);
2340 if (!p1) {
2341 mmap_unlock();
2342 return 0;
2344 host_end = host_start + qemu_host_page_size;
2345 p = p1;
2346 prot = 0;
2347 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2348 prot |= p->flags;
2349 p++;
2351 /* if the page was really writable, then we change its
2352 protection back to writable */
2353 if (prot & PAGE_WRITE_ORG) {
2354 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2355 if (!(p1[pindex].flags & PAGE_WRITE)) {
2356 mprotect((void *)g2h(host_start), qemu_host_page_size,
2357 (prot & PAGE_BITS) | PAGE_WRITE);
2358 p1[pindex].flags |= PAGE_WRITE;
2359 /* and since the content will be modified, we must invalidate
2360 the corresponding translated code. */
2361 tb_invalidate_phys_page(address, pc, puc);
2362 #ifdef DEBUG_TB_CHECK
2363 tb_invalidate_check(address);
2364 #endif
2365 mmap_unlock();
2366 return 1;
2369 mmap_unlock();
2370 return 0;
2373 static inline void tlb_set_dirty(CPUState *env,
2374 unsigned long addr, target_ulong vaddr)
2377 #endif /* defined(CONFIG_USER_ONLY) */
2379 #if !defined(CONFIG_USER_ONLY)
2381 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
2382 typedef struct subpage_t {
2383 target_phys_addr_t base;
2384 CPUReadMemoryFunc * const *mem_read[TARGET_PAGE_SIZE][4];
2385 CPUWriteMemoryFunc * const *mem_write[TARGET_PAGE_SIZE][4];
2386 void *opaque[TARGET_PAGE_SIZE][2][4];
2387 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
2388 } subpage_t;
2390 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2391 ram_addr_t memory, ram_addr_t region_offset);
2392 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2393 ram_addr_t orig_memory, ram_addr_t region_offset);
2394 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2395 need_subpage) \
2396 do { \
2397 if (addr > start_addr) \
2398 start_addr2 = 0; \
2399 else { \
2400 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2401 if (start_addr2 > 0) \
2402 need_subpage = 1; \
2405 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2406 end_addr2 = TARGET_PAGE_SIZE - 1; \
2407 else { \
2408 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2409 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2410 need_subpage = 1; \
2412 } while (0)
2414 /* register physical memory.
2415 For RAM, 'size' must be a multiple of the target page size.
2416 If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2417 io memory page. The address used when calling the IO function is
2418 the offset from the start of the region, plus region_offset. Both
2419 start_addr and region_offset are rounded down to a page boundary
2420 before calculating this offset. This should not be a problem unless
2421 the low bits of start_addr and region_offset differ. */
2422 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2423 ram_addr_t size,
2424 ram_addr_t phys_offset,
2425 ram_addr_t region_offset)
2427 target_phys_addr_t addr, end_addr;
2428 PhysPageDesc *p;
2429 CPUState *env;
2430 ram_addr_t orig_size = size;
2431 void *subpage;
2433 cpu_notify_set_memory(start_addr, size, phys_offset);
2435 if (phys_offset == IO_MEM_UNASSIGNED) {
2436 region_offset = start_addr;
2438 region_offset &= TARGET_PAGE_MASK;
2439 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2440 end_addr = start_addr + (target_phys_addr_t)size;
2441 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2442 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2443 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2444 ram_addr_t orig_memory = p->phys_offset;
2445 target_phys_addr_t start_addr2, end_addr2;
2446 int need_subpage = 0;
2448 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2449 need_subpage);
2450 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2451 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2452 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2453 &p->phys_offset, orig_memory,
2454 p->region_offset);
2455 } else {
2456 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2457 >> IO_MEM_SHIFT];
2459 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2460 region_offset);
2461 p->region_offset = 0;
2462 } else {
2463 p->phys_offset = phys_offset;
2464 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2465 (phys_offset & IO_MEM_ROMD))
2466 phys_offset += TARGET_PAGE_SIZE;
2468 } else {
2469 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2470 p->phys_offset = phys_offset;
2471 p->region_offset = region_offset;
2472 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2473 (phys_offset & IO_MEM_ROMD)) {
2474 phys_offset += TARGET_PAGE_SIZE;
2475 } else {
2476 target_phys_addr_t start_addr2, end_addr2;
2477 int need_subpage = 0;
2479 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2480 end_addr2, need_subpage);
2482 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2483 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2484 &p->phys_offset, IO_MEM_UNASSIGNED,
2485 addr & TARGET_PAGE_MASK);
2486 subpage_register(subpage, start_addr2, end_addr2,
2487 phys_offset, region_offset);
2488 p->region_offset = 0;
2492 region_offset += TARGET_PAGE_SIZE;
2495 /* since each CPU stores ram addresses in its TLB cache, we must
2496 reset the modified entries */
2497 /* XXX: slow ! */
2498 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2499 tlb_flush(env, 1);
2503 /* XXX: temporary until new memory mapping API */
2504 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2506 PhysPageDesc *p;
2508 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2509 if (!p)
2510 return IO_MEM_UNASSIGNED;
2511 return p->phys_offset;
2514 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2516 if (kvm_enabled())
2517 kvm_coalesce_mmio_region(addr, size);
2520 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2522 if (kvm_enabled())
2523 kvm_uncoalesce_mmio_region(addr, size);
2526 void qemu_flush_coalesced_mmio_buffer(void)
2528 if (kvm_enabled())
2529 kvm_flush_coalesced_mmio_buffer();
2532 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2534 RAMBlock *new_block;
2536 size = TARGET_PAGE_ALIGN(size);
2537 new_block = qemu_malloc(sizeof(*new_block));
2539 #if defined(TARGET_S390X) && defined(CONFIG_KVM)
2540 /* XXX S390 KVM requires the topmost vma of the RAM to be < 256GB */
2541 new_block->host = mmap((void*)0x1000000, size, PROT_EXEC|PROT_READ|PROT_WRITE,
2542 MAP_SHARED | MAP_ANONYMOUS, -1, 0);
2543 #else
2544 new_block->host = qemu_vmalloc(size);
2545 #endif
2546 #ifdef MADV_MERGEABLE
2547 madvise(new_block->host, size, MADV_MERGEABLE);
2548 #endif
2549 new_block->offset = last_ram_offset;
2550 new_block->length = size;
2552 new_block->next = ram_blocks;
2553 ram_blocks = new_block;
2555 phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2556 (last_ram_offset + size) >> TARGET_PAGE_BITS);
2557 memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2558 0xff, size >> TARGET_PAGE_BITS);
2560 last_ram_offset += size;
2562 if (kvm_enabled())
2563 kvm_setup_guest_memory(new_block->host, size);
2565 return new_block->offset;
2568 void qemu_ram_free(ram_addr_t addr)
2570 /* TODO: implement this. */
2573 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2574 With the exception of the softmmu code in this file, this should
2575 only be used for local memory (e.g. video ram) that the device owns,
2576 and knows it isn't going to access beyond the end of the block.
2578 It should not be used for general purpose DMA.
2579 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2581 void *qemu_get_ram_ptr(ram_addr_t addr)
2583 RAMBlock *prev;
2584 RAMBlock **prevp;
2585 RAMBlock *block;
2587 prev = NULL;
2588 prevp = &ram_blocks;
2589 block = ram_blocks;
2590 while (block && (block->offset > addr
2591 || block->offset + block->length <= addr)) {
2592 if (prev)
2593 prevp = &prev->next;
2594 prev = block;
2595 block = block->next;
2597 if (!block) {
2598 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2599 abort();
2601 /* Move this entry to to start of the list. */
2602 if (prev) {
2603 prev->next = block->next;
2604 block->next = *prevp;
2605 *prevp = block;
2607 return block->host + (addr - block->offset);
2610 /* Some of the softmmu routines need to translate from a host pointer
2611 (typically a TLB entry) back to a ram offset. */
2612 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2614 RAMBlock *prev;
2615 RAMBlock *block;
2616 uint8_t *host = ptr;
2618 prev = NULL;
2619 block = ram_blocks;
2620 while (block && (block->host > host
2621 || block->host + block->length <= host)) {
2622 prev = block;
2623 block = block->next;
2625 if (!block) {
2626 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2627 abort();
2629 return block->offset + (host - block->host);
2632 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2634 #ifdef DEBUG_UNASSIGNED
2635 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2636 #endif
2637 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2638 do_unassigned_access(addr, 0, 0, 0, 1);
2639 #endif
2640 return 0;
2643 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2645 #ifdef DEBUG_UNASSIGNED
2646 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2647 #endif
2648 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2649 do_unassigned_access(addr, 0, 0, 0, 2);
2650 #endif
2651 return 0;
2654 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2656 #ifdef DEBUG_UNASSIGNED
2657 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2658 #endif
2659 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2660 do_unassigned_access(addr, 0, 0, 0, 4);
2661 #endif
2662 return 0;
2665 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2667 #ifdef DEBUG_UNASSIGNED
2668 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2669 #endif
2670 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2671 do_unassigned_access(addr, 1, 0, 0, 1);
2672 #endif
2675 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2677 #ifdef DEBUG_UNASSIGNED
2678 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2679 #endif
2680 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2681 do_unassigned_access(addr, 1, 0, 0, 2);
2682 #endif
2685 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2687 #ifdef DEBUG_UNASSIGNED
2688 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2689 #endif
2690 #if defined(TARGET_SPARC) || defined(TARGET_MICROBLAZE)
2691 do_unassigned_access(addr, 1, 0, 0, 4);
2692 #endif
2695 static CPUReadMemoryFunc * const unassigned_mem_read[3] = {
2696 unassigned_mem_readb,
2697 unassigned_mem_readw,
2698 unassigned_mem_readl,
2701 static CPUWriteMemoryFunc * const unassigned_mem_write[3] = {
2702 unassigned_mem_writeb,
2703 unassigned_mem_writew,
2704 unassigned_mem_writel,
2707 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2708 uint32_t val)
2710 int dirty_flags;
2711 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2712 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2713 #if !defined(CONFIG_USER_ONLY)
2714 tb_invalidate_phys_page_fast(ram_addr, 1);
2715 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2716 #endif
2718 stb_p(qemu_get_ram_ptr(ram_addr), val);
2719 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2720 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2721 /* we remove the notdirty callback only if the code has been
2722 flushed */
2723 if (dirty_flags == 0xff)
2724 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2727 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2728 uint32_t val)
2730 int dirty_flags;
2731 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2732 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2733 #if !defined(CONFIG_USER_ONLY)
2734 tb_invalidate_phys_page_fast(ram_addr, 2);
2735 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2736 #endif
2738 stw_p(qemu_get_ram_ptr(ram_addr), val);
2739 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2740 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2741 /* we remove the notdirty callback only if the code has been
2742 flushed */
2743 if (dirty_flags == 0xff)
2744 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2747 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2748 uint32_t val)
2750 int dirty_flags;
2751 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2752 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2753 #if !defined(CONFIG_USER_ONLY)
2754 tb_invalidate_phys_page_fast(ram_addr, 4);
2755 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2756 #endif
2758 stl_p(qemu_get_ram_ptr(ram_addr), val);
2759 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2760 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2761 /* we remove the notdirty callback only if the code has been
2762 flushed */
2763 if (dirty_flags == 0xff)
2764 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2767 static CPUReadMemoryFunc * const error_mem_read[3] = {
2768 NULL, /* never used */
2769 NULL, /* never used */
2770 NULL, /* never used */
2773 static CPUWriteMemoryFunc * const notdirty_mem_write[3] = {
2774 notdirty_mem_writeb,
2775 notdirty_mem_writew,
2776 notdirty_mem_writel,
2779 /* Generate a debug exception if a watchpoint has been hit. */
2780 static void check_watchpoint(int offset, int len_mask, int flags)
2782 CPUState *env = cpu_single_env;
2783 target_ulong pc, cs_base;
2784 TranslationBlock *tb;
2785 target_ulong vaddr;
2786 CPUWatchpoint *wp;
2787 int cpu_flags;
2789 if (env->watchpoint_hit) {
2790 /* We re-entered the check after replacing the TB. Now raise
2791 * the debug interrupt so that is will trigger after the
2792 * current instruction. */
2793 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2794 return;
2796 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2797 QTAILQ_FOREACH(wp, &env->watchpoints, entry) {
2798 if ((vaddr == (wp->vaddr & len_mask) ||
2799 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2800 wp->flags |= BP_WATCHPOINT_HIT;
2801 if (!env->watchpoint_hit) {
2802 env->watchpoint_hit = wp;
2803 tb = tb_find_pc(env->mem_io_pc);
2804 if (!tb) {
2805 cpu_abort(env, "check_watchpoint: could not find TB for "
2806 "pc=%p", (void *)env->mem_io_pc);
2808 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2809 tb_phys_invalidate(tb, -1);
2810 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2811 env->exception_index = EXCP_DEBUG;
2812 } else {
2813 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2814 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2816 cpu_resume_from_signal(env, NULL);
2818 } else {
2819 wp->flags &= ~BP_WATCHPOINT_HIT;
2824 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2825 so these check for a hit then pass through to the normal out-of-line
2826 phys routines. */
2827 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2829 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2830 return ldub_phys(addr);
2833 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2835 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2836 return lduw_phys(addr);
2839 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2841 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2842 return ldl_phys(addr);
2845 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2846 uint32_t val)
2848 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2849 stb_phys(addr, val);
2852 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2853 uint32_t val)
2855 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2856 stw_phys(addr, val);
2859 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2860 uint32_t val)
2862 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2863 stl_phys(addr, val);
2866 static CPUReadMemoryFunc * const watch_mem_read[3] = {
2867 watch_mem_readb,
2868 watch_mem_readw,
2869 watch_mem_readl,
2872 static CPUWriteMemoryFunc * const watch_mem_write[3] = {
2873 watch_mem_writeb,
2874 watch_mem_writew,
2875 watch_mem_writel,
2878 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2879 unsigned int len)
2881 uint32_t ret;
2882 unsigned int idx;
2884 idx = SUBPAGE_IDX(addr);
2885 #if defined(DEBUG_SUBPAGE)
2886 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2887 mmio, len, addr, idx);
2888 #endif
2889 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2890 addr + mmio->region_offset[idx][0][len]);
2892 return ret;
2895 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2896 uint32_t value, unsigned int len)
2898 unsigned int idx;
2900 idx = SUBPAGE_IDX(addr);
2901 #if defined(DEBUG_SUBPAGE)
2902 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2903 mmio, len, addr, idx, value);
2904 #endif
2905 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2906 addr + mmio->region_offset[idx][1][len],
2907 value);
2910 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2912 #if defined(DEBUG_SUBPAGE)
2913 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2914 #endif
2916 return subpage_readlen(opaque, addr, 0);
2919 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2920 uint32_t value)
2922 #if defined(DEBUG_SUBPAGE)
2923 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2924 #endif
2925 subpage_writelen(opaque, addr, value, 0);
2928 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2930 #if defined(DEBUG_SUBPAGE)
2931 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2932 #endif
2934 return subpage_readlen(opaque, addr, 1);
2937 static void subpage_writew (void *opaque, target_phys_addr_t addr,
2938 uint32_t value)
2940 #if defined(DEBUG_SUBPAGE)
2941 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2942 #endif
2943 subpage_writelen(opaque, addr, value, 1);
2946 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2948 #if defined(DEBUG_SUBPAGE)
2949 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2950 #endif
2952 return subpage_readlen(opaque, addr, 2);
2955 static void subpage_writel (void *opaque,
2956 target_phys_addr_t addr, uint32_t value)
2958 #if defined(DEBUG_SUBPAGE)
2959 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2960 #endif
2961 subpage_writelen(opaque, addr, value, 2);
2964 static CPUReadMemoryFunc * const subpage_read[] = {
2965 &subpage_readb,
2966 &subpage_readw,
2967 &subpage_readl,
2970 static CPUWriteMemoryFunc * const subpage_write[] = {
2971 &subpage_writeb,
2972 &subpage_writew,
2973 &subpage_writel,
2976 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2977 ram_addr_t memory, ram_addr_t region_offset)
2979 int idx, eidx;
2980 unsigned int i;
2982 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2983 return -1;
2984 idx = SUBPAGE_IDX(start);
2985 eidx = SUBPAGE_IDX(end);
2986 #if defined(DEBUG_SUBPAGE)
2987 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
2988 mmio, start, end, idx, eidx, memory);
2989 #endif
2990 memory >>= IO_MEM_SHIFT;
2991 for (; idx <= eidx; idx++) {
2992 for (i = 0; i < 4; i++) {
2993 if (io_mem_read[memory][i]) {
2994 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2995 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
2996 mmio->region_offset[idx][0][i] = region_offset;
2998 if (io_mem_write[memory][i]) {
2999 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
3000 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
3001 mmio->region_offset[idx][1][i] = region_offset;
3006 return 0;
3009 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3010 ram_addr_t orig_memory, ram_addr_t region_offset)
3012 subpage_t *mmio;
3013 int subpage_memory;
3015 mmio = qemu_mallocz(sizeof(subpage_t));
3017 mmio->base = base;
3018 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3019 #if defined(DEBUG_SUBPAGE)
3020 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3021 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3022 #endif
3023 *phys = subpage_memory | IO_MEM_SUBPAGE;
3024 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3025 region_offset);
3027 return mmio;
3030 static int get_free_io_mem_idx(void)
3032 int i;
3034 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3035 if (!io_mem_used[i]) {
3036 io_mem_used[i] = 1;
3037 return i;
3039 fprintf(stderr, "RAN out out io_mem_idx, max %d !\n", IO_MEM_NB_ENTRIES);
3040 return -1;
3043 /* mem_read and mem_write are arrays of functions containing the
3044 function to access byte (index 0), word (index 1) and dword (index
3045 2). Functions can be omitted with a NULL function pointer.
3046 If io_index is non zero, the corresponding io zone is
3047 modified. If it is zero, a new io zone is allocated. The return
3048 value can be used with cpu_register_physical_memory(). (-1) is
3049 returned if error. */
3050 static int cpu_register_io_memory_fixed(int io_index,
3051 CPUReadMemoryFunc * const *mem_read,
3052 CPUWriteMemoryFunc * const *mem_write,
3053 void *opaque)
3055 int i, subwidth = 0;
3057 if (io_index <= 0) {
3058 io_index = get_free_io_mem_idx();
3059 if (io_index == -1)
3060 return io_index;
3061 } else {
3062 io_index >>= IO_MEM_SHIFT;
3063 if (io_index >= IO_MEM_NB_ENTRIES)
3064 return -1;
3067 for(i = 0;i < 3; i++) {
3068 if (!mem_read[i] || !mem_write[i])
3069 subwidth = IO_MEM_SUBWIDTH;
3070 io_mem_read[io_index][i] = mem_read[i];
3071 io_mem_write[io_index][i] = mem_write[i];
3073 io_mem_opaque[io_index] = opaque;
3074 return (io_index << IO_MEM_SHIFT) | subwidth;
3077 int cpu_register_io_memory(CPUReadMemoryFunc * const *mem_read,
3078 CPUWriteMemoryFunc * const *mem_write,
3079 void *opaque)
3081 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3084 void cpu_unregister_io_memory(int io_table_address)
3086 int i;
3087 int io_index = io_table_address >> IO_MEM_SHIFT;
3089 for (i=0;i < 3; i++) {
3090 io_mem_read[io_index][i] = unassigned_mem_read[i];
3091 io_mem_write[io_index][i] = unassigned_mem_write[i];
3093 io_mem_opaque[io_index] = NULL;
3094 io_mem_used[io_index] = 0;
3097 static void io_mem_init(void)
3099 int i;
3101 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3102 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3103 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3104 for (i=0; i<5; i++)
3105 io_mem_used[i] = 1;
3107 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3108 watch_mem_write, NULL);
3111 #endif /* !defined(CONFIG_USER_ONLY) */
3113 /* physical memory access (slow version, mainly for debug) */
3114 #if defined(CONFIG_USER_ONLY)
3115 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3116 uint8_t *buf, int len, int is_write)
3118 int l, flags;
3119 target_ulong page;
3120 void * p;
3122 while (len > 0) {
3123 page = addr & TARGET_PAGE_MASK;
3124 l = (page + TARGET_PAGE_SIZE) - addr;
3125 if (l > len)
3126 l = len;
3127 flags = page_get_flags(page);
3128 if (!(flags & PAGE_VALID))
3129 return -1;
3130 if (is_write) {
3131 if (!(flags & PAGE_WRITE))
3132 return -1;
3133 /* XXX: this code should not depend on lock_user */
3134 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3135 return -1;
3136 memcpy(p, buf, l);
3137 unlock_user(p, addr, l);
3138 } else {
3139 if (!(flags & PAGE_READ))
3140 return -1;
3141 /* XXX: this code should not depend on lock_user */
3142 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3143 return -1;
3144 memcpy(buf, p, l);
3145 unlock_user(p, addr, 0);
3147 len -= l;
3148 buf += l;
3149 addr += l;
3151 return 0;
3154 #else
3155 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3156 int len, int is_write)
3158 int l, io_index;
3159 uint8_t *ptr;
3160 uint32_t val;
3161 target_phys_addr_t page;
3162 unsigned long pd;
3163 PhysPageDesc *p;
3165 while (len > 0) {
3166 page = addr & TARGET_PAGE_MASK;
3167 l = (page + TARGET_PAGE_SIZE) - addr;
3168 if (l > len)
3169 l = len;
3170 p = phys_page_find(page >> TARGET_PAGE_BITS);
3171 if (!p) {
3172 pd = IO_MEM_UNASSIGNED;
3173 } else {
3174 pd = p->phys_offset;
3177 if (is_write) {
3178 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3179 target_phys_addr_t addr1 = addr;
3180 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3181 if (p)
3182 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3183 /* XXX: could force cpu_single_env to NULL to avoid
3184 potential bugs */
3185 if (l >= 4 && ((addr1 & 3) == 0)) {
3186 /* 32 bit write access */
3187 val = ldl_p(buf);
3188 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3189 l = 4;
3190 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3191 /* 16 bit write access */
3192 val = lduw_p(buf);
3193 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3194 l = 2;
3195 } else {
3196 /* 8 bit write access */
3197 val = ldub_p(buf);
3198 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3199 l = 1;
3201 } else {
3202 unsigned long addr1;
3203 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3204 /* RAM case */
3205 ptr = qemu_get_ram_ptr(addr1);
3206 memcpy(ptr, buf, l);
3207 if (!cpu_physical_memory_is_dirty(addr1)) {
3208 /* invalidate code */
3209 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3210 /* set dirty bit */
3211 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3212 (0xff & ~CODE_DIRTY_FLAG);
3215 } else {
3216 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3217 !(pd & IO_MEM_ROMD)) {
3218 target_phys_addr_t addr1 = addr;
3219 /* I/O case */
3220 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3221 if (p)
3222 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3223 if (l >= 4 && ((addr1 & 3) == 0)) {
3224 /* 32 bit read access */
3225 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3226 stl_p(buf, val);
3227 l = 4;
3228 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3229 /* 16 bit read access */
3230 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3231 stw_p(buf, val);
3232 l = 2;
3233 } else {
3234 /* 8 bit read access */
3235 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3236 stb_p(buf, val);
3237 l = 1;
3239 } else {
3240 /* RAM case */
3241 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3242 (addr & ~TARGET_PAGE_MASK);
3243 memcpy(buf, ptr, l);
3246 len -= l;
3247 buf += l;
3248 addr += l;
3252 /* used for ROM loading : can write in RAM and ROM */
3253 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3254 const uint8_t *buf, int len)
3256 int l;
3257 uint8_t *ptr;
3258 target_phys_addr_t page;
3259 unsigned long pd;
3260 PhysPageDesc *p;
3262 while (len > 0) {
3263 page = addr & TARGET_PAGE_MASK;
3264 l = (page + TARGET_PAGE_SIZE) - addr;
3265 if (l > len)
3266 l = len;
3267 p = phys_page_find(page >> TARGET_PAGE_BITS);
3268 if (!p) {
3269 pd = IO_MEM_UNASSIGNED;
3270 } else {
3271 pd = p->phys_offset;
3274 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3275 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3276 !(pd & IO_MEM_ROMD)) {
3277 /* do nothing */
3278 } else {
3279 unsigned long addr1;
3280 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3281 /* ROM/RAM case */
3282 ptr = qemu_get_ram_ptr(addr1);
3283 memcpy(ptr, buf, l);
3285 len -= l;
3286 buf += l;
3287 addr += l;
3291 typedef struct {
3292 void *buffer;
3293 target_phys_addr_t addr;
3294 target_phys_addr_t len;
3295 } BounceBuffer;
3297 static BounceBuffer bounce;
3299 typedef struct MapClient {
3300 void *opaque;
3301 void (*callback)(void *opaque);
3302 QLIST_ENTRY(MapClient) link;
3303 } MapClient;
3305 static QLIST_HEAD(map_client_list, MapClient) map_client_list
3306 = QLIST_HEAD_INITIALIZER(map_client_list);
3308 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3310 MapClient *client = qemu_malloc(sizeof(*client));
3312 client->opaque = opaque;
3313 client->callback = callback;
3314 QLIST_INSERT_HEAD(&map_client_list, client, link);
3315 return client;
3318 void cpu_unregister_map_client(void *_client)
3320 MapClient *client = (MapClient *)_client;
3322 QLIST_REMOVE(client, link);
3323 qemu_free(client);
3326 static void cpu_notify_map_clients(void)
3328 MapClient *client;
3330 while (!QLIST_EMPTY(&map_client_list)) {
3331 client = QLIST_FIRST(&map_client_list);
3332 client->callback(client->opaque);
3333 cpu_unregister_map_client(client);
3337 /* Map a physical memory region into a host virtual address.
3338 * May map a subset of the requested range, given by and returned in *plen.
3339 * May return NULL if resources needed to perform the mapping are exhausted.
3340 * Use only for reads OR writes - not for read-modify-write operations.
3341 * Use cpu_register_map_client() to know when retrying the map operation is
3342 * likely to succeed.
3344 void *cpu_physical_memory_map(target_phys_addr_t addr,
3345 target_phys_addr_t *plen,
3346 int is_write)
3348 target_phys_addr_t len = *plen;
3349 target_phys_addr_t done = 0;
3350 int l;
3351 uint8_t *ret = NULL;
3352 uint8_t *ptr;
3353 target_phys_addr_t page;
3354 unsigned long pd;
3355 PhysPageDesc *p;
3356 unsigned long addr1;
3358 while (len > 0) {
3359 page = addr & TARGET_PAGE_MASK;
3360 l = (page + TARGET_PAGE_SIZE) - addr;
3361 if (l > len)
3362 l = len;
3363 p = phys_page_find(page >> TARGET_PAGE_BITS);
3364 if (!p) {
3365 pd = IO_MEM_UNASSIGNED;
3366 } else {
3367 pd = p->phys_offset;
3370 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3371 if (done || bounce.buffer) {
3372 break;
3374 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3375 bounce.addr = addr;
3376 bounce.len = l;
3377 if (!is_write) {
3378 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3380 ptr = bounce.buffer;
3381 } else {
3382 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3383 ptr = qemu_get_ram_ptr(addr1);
3385 if (!done) {
3386 ret = ptr;
3387 } else if (ret + done != ptr) {
3388 break;
3391 len -= l;
3392 addr += l;
3393 done += l;
3395 *plen = done;
3396 return ret;
3399 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3400 * Will also mark the memory as dirty if is_write == 1. access_len gives
3401 * the amount of memory that was actually read or written by the caller.
3403 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3404 int is_write, target_phys_addr_t access_len)
3406 if (buffer != bounce.buffer) {
3407 if (is_write) {
3408 ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3409 while (access_len) {
3410 unsigned l;
3411 l = TARGET_PAGE_SIZE;
3412 if (l > access_len)
3413 l = access_len;
3414 if (!cpu_physical_memory_is_dirty(addr1)) {
3415 /* invalidate code */
3416 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3417 /* set dirty bit */
3418 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3419 (0xff & ~CODE_DIRTY_FLAG);
3421 addr1 += l;
3422 access_len -= l;
3425 return;
3427 if (is_write) {
3428 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3430 qemu_vfree(bounce.buffer);
3431 bounce.buffer = NULL;
3432 cpu_notify_map_clients();
3435 /* warning: addr must be aligned */
3436 uint32_t ldl_phys(target_phys_addr_t addr)
3438 int io_index;
3439 uint8_t *ptr;
3440 uint32_t val;
3441 unsigned long pd;
3442 PhysPageDesc *p;
3444 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3445 if (!p) {
3446 pd = IO_MEM_UNASSIGNED;
3447 } else {
3448 pd = p->phys_offset;
3451 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3452 !(pd & IO_MEM_ROMD)) {
3453 /* I/O case */
3454 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3455 if (p)
3456 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3457 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3458 } else {
3459 /* RAM case */
3460 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3461 (addr & ~TARGET_PAGE_MASK);
3462 val = ldl_p(ptr);
3464 return val;
3467 /* warning: addr must be aligned */
3468 uint64_t ldq_phys(target_phys_addr_t addr)
3470 int io_index;
3471 uint8_t *ptr;
3472 uint64_t val;
3473 unsigned long pd;
3474 PhysPageDesc *p;
3476 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3477 if (!p) {
3478 pd = IO_MEM_UNASSIGNED;
3479 } else {
3480 pd = p->phys_offset;
3483 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3484 !(pd & IO_MEM_ROMD)) {
3485 /* I/O case */
3486 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3487 if (p)
3488 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3489 #ifdef TARGET_WORDS_BIGENDIAN
3490 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3491 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3492 #else
3493 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3494 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3495 #endif
3496 } else {
3497 /* RAM case */
3498 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3499 (addr & ~TARGET_PAGE_MASK);
3500 val = ldq_p(ptr);
3502 return val;
3505 /* XXX: optimize */
3506 uint32_t ldub_phys(target_phys_addr_t addr)
3508 uint8_t val;
3509 cpu_physical_memory_read(addr, &val, 1);
3510 return val;
3513 /* XXX: optimize */
3514 uint32_t lduw_phys(target_phys_addr_t addr)
3516 uint16_t val;
3517 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3518 return tswap16(val);
3521 /* warning: addr must be aligned. The ram page is not masked as dirty
3522 and the code inside is not invalidated. It is useful if the dirty
3523 bits are used to track modified PTEs */
3524 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3526 int io_index;
3527 uint8_t *ptr;
3528 unsigned long pd;
3529 PhysPageDesc *p;
3531 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3532 if (!p) {
3533 pd = IO_MEM_UNASSIGNED;
3534 } else {
3535 pd = p->phys_offset;
3538 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3539 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3540 if (p)
3541 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3542 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3543 } else {
3544 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3545 ptr = qemu_get_ram_ptr(addr1);
3546 stl_p(ptr, val);
3548 if (unlikely(in_migration)) {
3549 if (!cpu_physical_memory_is_dirty(addr1)) {
3550 /* invalidate code */
3551 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3552 /* set dirty bit */
3553 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3554 (0xff & ~CODE_DIRTY_FLAG);
3560 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3562 int io_index;
3563 uint8_t *ptr;
3564 unsigned long pd;
3565 PhysPageDesc *p;
3567 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3568 if (!p) {
3569 pd = IO_MEM_UNASSIGNED;
3570 } else {
3571 pd = p->phys_offset;
3574 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3575 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3576 if (p)
3577 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3578 #ifdef TARGET_WORDS_BIGENDIAN
3579 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3580 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3581 #else
3582 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3583 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3584 #endif
3585 } else {
3586 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3587 (addr & ~TARGET_PAGE_MASK);
3588 stq_p(ptr, val);
3592 /* warning: addr must be aligned */
3593 void stl_phys(target_phys_addr_t addr, uint32_t val)
3595 int io_index;
3596 uint8_t *ptr;
3597 unsigned long pd;
3598 PhysPageDesc *p;
3600 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3601 if (!p) {
3602 pd = IO_MEM_UNASSIGNED;
3603 } else {
3604 pd = p->phys_offset;
3607 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3608 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3609 if (p)
3610 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3611 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3612 } else {
3613 unsigned long addr1;
3614 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3615 /* RAM case */
3616 ptr = qemu_get_ram_ptr(addr1);
3617 stl_p(ptr, val);
3618 if (!cpu_physical_memory_is_dirty(addr1)) {
3619 /* invalidate code */
3620 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3621 /* set dirty bit */
3622 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3623 (0xff & ~CODE_DIRTY_FLAG);
3628 /* XXX: optimize */
3629 void stb_phys(target_phys_addr_t addr, uint32_t val)
3631 uint8_t v = val;
3632 cpu_physical_memory_write(addr, &v, 1);
3635 /* XXX: optimize */
3636 void stw_phys(target_phys_addr_t addr, uint32_t val)
3638 uint16_t v = tswap16(val);
3639 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3642 /* XXX: optimize */
3643 void stq_phys(target_phys_addr_t addr, uint64_t val)
3645 val = tswap64(val);
3646 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3649 /* virtual memory access for debug (includes writing to ROM) */
3650 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3651 uint8_t *buf, int len, int is_write)
3653 int l;
3654 target_phys_addr_t phys_addr;
3655 target_ulong page;
3657 while (len > 0) {
3658 page = addr & TARGET_PAGE_MASK;
3659 phys_addr = cpu_get_phys_page_debug(env, page);
3660 /* if no physical page mapped, return an error */
3661 if (phys_addr == -1)
3662 return -1;
3663 l = (page + TARGET_PAGE_SIZE) - addr;
3664 if (l > len)
3665 l = len;
3666 phys_addr += (addr & ~TARGET_PAGE_MASK);
3667 if (is_write)
3668 cpu_physical_memory_write_rom(phys_addr, buf, l);
3669 else
3670 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3671 len -= l;
3672 buf += l;
3673 addr += l;
3675 return 0;
3677 #endif
3679 /* in deterministic execution mode, instructions doing device I/Os
3680 must be at the end of the TB */
3681 void cpu_io_recompile(CPUState *env, void *retaddr)
3683 TranslationBlock *tb;
3684 uint32_t n, cflags;
3685 target_ulong pc, cs_base;
3686 uint64_t flags;
3688 tb = tb_find_pc((unsigned long)retaddr);
3689 if (!tb) {
3690 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3691 retaddr);
3693 n = env->icount_decr.u16.low + tb->icount;
3694 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3695 /* Calculate how many instructions had been executed before the fault
3696 occurred. */
3697 n = n - env->icount_decr.u16.low;
3698 /* Generate a new TB ending on the I/O insn. */
3699 n++;
3700 /* On MIPS and SH, delay slot instructions can only be restarted if
3701 they were already the first instruction in the TB. If this is not
3702 the first instruction in a TB then re-execute the preceding
3703 branch. */
3704 #if defined(TARGET_MIPS)
3705 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3706 env->active_tc.PC -= 4;
3707 env->icount_decr.u16.low++;
3708 env->hflags &= ~MIPS_HFLAG_BMASK;
3710 #elif defined(TARGET_SH4)
3711 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3712 && n > 1) {
3713 env->pc -= 2;
3714 env->icount_decr.u16.low++;
3715 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3717 #endif
3718 /* This should never happen. */
3719 if (n > CF_COUNT_MASK)
3720 cpu_abort(env, "TB too big during recompile");
3722 cflags = n | CF_LAST_IO;
3723 pc = tb->pc;
3724 cs_base = tb->cs_base;
3725 flags = tb->flags;
3726 tb_phys_invalidate(tb, -1);
3727 /* FIXME: In theory this could raise an exception. In practice
3728 we have already translated the block once so it's probably ok. */
3729 tb_gen_code(env, pc, cs_base, flags, cflags);
3730 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3731 the first in the TB) then we end up generating a whole new TB and
3732 repeating the fault, which is horribly inefficient.
3733 Better would be to execute just this insn uncached, or generate a
3734 second new TB. */
3735 cpu_resume_from_signal(env, NULL);
3738 void dump_exec_info(FILE *f,
3739 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3741 int i, target_code_size, max_target_code_size;
3742 int direct_jmp_count, direct_jmp2_count, cross_page;
3743 TranslationBlock *tb;
3745 target_code_size = 0;
3746 max_target_code_size = 0;
3747 cross_page = 0;
3748 direct_jmp_count = 0;
3749 direct_jmp2_count = 0;
3750 for(i = 0; i < nb_tbs; i++) {
3751 tb = &tbs[i];
3752 target_code_size += tb->size;
3753 if (tb->size > max_target_code_size)
3754 max_target_code_size = tb->size;
3755 if (tb->page_addr[1] != -1)
3756 cross_page++;
3757 if (tb->tb_next_offset[0] != 0xffff) {
3758 direct_jmp_count++;
3759 if (tb->tb_next_offset[1] != 0xffff) {
3760 direct_jmp2_count++;
3764 /* XXX: avoid using doubles ? */
3765 cpu_fprintf(f, "Translation buffer state:\n");
3766 cpu_fprintf(f, "gen code size %ld/%ld\n",
3767 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3768 cpu_fprintf(f, "TB count %d/%d\n",
3769 nb_tbs, code_gen_max_blocks);
3770 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3771 nb_tbs ? target_code_size / nb_tbs : 0,
3772 max_target_code_size);
3773 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3774 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3775 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3776 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3777 cross_page,
3778 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3779 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3780 direct_jmp_count,
3781 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3782 direct_jmp2_count,
3783 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3784 cpu_fprintf(f, "\nStatistics:\n");
3785 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3786 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3787 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3788 tcg_dump_info(f, cpu_fprintf);
3791 #if !defined(CONFIG_USER_ONLY)
3793 #define MMUSUFFIX _cmmu
3794 #define GETPC() NULL
3795 #define env cpu_single_env
3796 #define SOFTMMU_CODE_ACCESS
3798 #define SHIFT 0
3799 #include "softmmu_template.h"
3801 #define SHIFT 1
3802 #include "softmmu_template.h"
3804 #define SHIFT 2
3805 #include "softmmu_template.h"
3807 #define SHIFT 3
3808 #include "softmmu_template.h"
3810 #undef env
3812 #endif