Merge branch 'stable-0.11' of git://git.sv.gnu.org/qemu into stable-0.11
[qemu-kvm/fedora.git] / exec.c
blob0655b4b55dde41336e1f3906174c846124d58a7d
1 /*
2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "config.h"
20 #ifdef _WIN32
21 #include <windows.h>
22 #else
23 #include <sys/types.h>
24 #include <sys/mman.h>
25 #endif
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <stdarg.h>
29 #include <string.h>
30 #include <errno.h>
31 #include <unistd.h>
32 #include <inttypes.h>
34 #include "cpu.h"
35 #include "exec-all.h"
36 #include "qemu-common.h"
37 #include "cache-utils.h"
39 #if !defined(TARGET_IA64)
40 #include "tcg.h"
41 #endif
42 #include "qemu-kvm.h"
44 #include "hw/hw.h"
45 #include "osdep.h"
46 #include "kvm.h"
47 #if defined(CONFIG_USER_ONLY)
48 #include <qemu.h>
49 #endif
51 //#define DEBUG_TB_INVALIDATE
52 //#define DEBUG_FLUSH
53 //#define DEBUG_TLB
54 //#define DEBUG_UNASSIGNED
56 /* make various TB consistency checks */
57 //#define DEBUG_TB_CHECK
58 //#define DEBUG_TLB_CHECK
60 //#define DEBUG_IOPORT
61 //#define DEBUG_SUBPAGE
63 #if !defined(CONFIG_USER_ONLY)
64 /* TB consistency checks only implemented for usermode emulation. */
65 #undef DEBUG_TB_CHECK
66 #endif
68 #define SMC_BITMAP_USE_THRESHOLD 10
70 #if defined(TARGET_SPARC64)
71 #define TARGET_PHYS_ADDR_SPACE_BITS 41
72 #elif defined(TARGET_SPARC)
73 #define TARGET_PHYS_ADDR_SPACE_BITS 36
74 #elif defined(TARGET_ALPHA)
75 #define TARGET_PHYS_ADDR_SPACE_BITS 42
76 #define TARGET_VIRT_ADDR_SPACE_BITS 42
77 #elif defined(TARGET_PPC64)
78 #define TARGET_PHYS_ADDR_SPACE_BITS 42
79 #elif defined(TARGET_X86_64) && !defined(CONFIG_KQEMU)
80 #define TARGET_PHYS_ADDR_SPACE_BITS 42
81 #elif defined(TARGET_I386) && !defined(CONFIG_KQEMU)
82 #define TARGET_PHYS_ADDR_SPACE_BITS 36
83 #elif defined(TARGET_IA64)
84 #define TARGET_PHYS_ADDR_SPACE_BITS 36
85 #else
86 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
87 #define TARGET_PHYS_ADDR_SPACE_BITS 32
88 #endif
90 static TranslationBlock *tbs;
91 int code_gen_max_blocks;
92 TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
93 static int nb_tbs;
94 /* any access to the tbs or the page table must use this lock */
95 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
97 #if defined(__arm__) || defined(__sparc_v9__)
98 /* The prologue must be reachable with a direct jump. ARM and Sparc64
99 have limited branch ranges (possibly also PPC) so place it in a
100 section close to code segment. */
101 #define code_gen_section \
102 __attribute__((__section__(".gen_code"))) \
103 __attribute__((aligned (32)))
104 #elif defined(_WIN32)
105 /* Maximum alignment for Win32 is 16. */
106 #define code_gen_section \
107 __attribute__((aligned (16)))
108 #else
109 #define code_gen_section \
110 __attribute__((aligned (32)))
111 #endif
113 uint8_t code_gen_prologue[1024] code_gen_section;
114 static uint8_t *code_gen_buffer;
115 static unsigned long code_gen_buffer_size;
116 /* threshold to flush the translated code buffer */
117 static unsigned long code_gen_buffer_max_size;
118 uint8_t *code_gen_ptr;
120 #if !defined(CONFIG_USER_ONLY)
121 int phys_ram_fd;
122 uint8_t *phys_ram_dirty;
123 uint8_t *bios_mem;
124 static int in_migration;
126 typedef struct RAMBlock {
127 uint8_t *host;
128 ram_addr_t offset;
129 ram_addr_t length;
130 struct RAMBlock *next;
131 } RAMBlock;
133 static RAMBlock *ram_blocks;
134 /* TODO: When we implement (and use) ram deallocation (e.g. for hotplug)
135 then we can no longer assume contiguous ram offsets, and external uses
136 of this variable will break. */
137 ram_addr_t last_ram_offset;
138 #endif
140 CPUState *first_cpu;
141 /* current CPU in the current thread. It is only valid inside
142 cpu_exec() */
143 CPUState *cpu_single_env;
144 /* 0 = Do not count executed instructions.
145 1 = Precise instruction counting.
146 2 = Adaptive rate instruction counting. */
147 int use_icount = 0;
148 /* Current instruction counter. While executing translated code this may
149 include some instructions that have not yet been executed. */
150 int64_t qemu_icount;
152 typedef struct PageDesc {
153 /* list of TBs intersecting this ram page */
154 TranslationBlock *first_tb;
155 /* in order to optimize self modifying code, we count the number
156 of lookups we do to a given page to use a bitmap */
157 unsigned int code_write_count;
158 uint8_t *code_bitmap;
159 #if defined(CONFIG_USER_ONLY)
160 unsigned long flags;
161 #endif
162 } PageDesc;
164 typedef struct PhysPageDesc {
165 /* offset in host memory of the page + io_index in the low bits */
166 ram_addr_t phys_offset;
167 ram_addr_t region_offset;
168 } PhysPageDesc;
170 #define L2_BITS 10
171 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
172 /* XXX: this is a temporary hack for alpha target.
173 * In the future, this is to be replaced by a multi-level table
174 * to actually be able to handle the complete 64 bits address space.
176 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
177 #else
178 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
179 #endif
181 #define L1_SIZE (1 << L1_BITS)
182 #define L2_SIZE (1 << L2_BITS)
184 unsigned long qemu_real_host_page_size;
185 unsigned long qemu_host_page_bits;
186 unsigned long qemu_host_page_size;
187 unsigned long qemu_host_page_mask;
189 /* XXX: for system emulation, it could just be an array */
190 static PageDesc *l1_map[L1_SIZE];
191 static PhysPageDesc **l1_phys_map;
193 #if !defined(CONFIG_USER_ONLY)
194 static void io_mem_init(void);
196 /* io memory support */
197 CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
198 CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
199 void *io_mem_opaque[IO_MEM_NB_ENTRIES];
200 static char io_mem_used[IO_MEM_NB_ENTRIES];
201 static int io_mem_watch;
202 #endif
204 /* log support */
205 static const char *logfilename = "/tmp/qemu.log";
206 FILE *logfile;
207 int loglevel;
208 static int log_append = 0;
210 /* statistics */
211 static int tlb_flush_count;
212 static int tb_flush_count;
213 static int tb_phys_invalidate_count;
215 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
216 typedef struct subpage_t {
217 target_phys_addr_t base;
218 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
219 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
220 void *opaque[TARGET_PAGE_SIZE][2][4];
221 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
222 } subpage_t;
224 #ifdef _WIN32
225 static void map_exec(void *addr, long size)
227 DWORD old_protect;
228 VirtualProtect(addr, size,
229 PAGE_EXECUTE_READWRITE, &old_protect);
232 #else
233 static void map_exec(void *addr, long size)
235 unsigned long start, end, page_size;
237 page_size = getpagesize();
238 start = (unsigned long)addr;
239 start &= ~(page_size - 1);
241 end = (unsigned long)addr + size;
242 end += page_size - 1;
243 end &= ~(page_size - 1);
245 mprotect((void *)start, end - start,
246 PROT_READ | PROT_WRITE | PROT_EXEC);
248 #endif
250 static void page_init(void)
252 /* NOTE: we can always suppose that qemu_host_page_size >=
253 TARGET_PAGE_SIZE */
254 #ifdef _WIN32
256 SYSTEM_INFO system_info;
258 GetSystemInfo(&system_info);
259 qemu_real_host_page_size = system_info.dwPageSize;
261 #else
262 qemu_real_host_page_size = getpagesize();
263 #endif
264 if (qemu_host_page_size == 0)
265 qemu_host_page_size = qemu_real_host_page_size;
266 if (qemu_host_page_size < TARGET_PAGE_SIZE)
267 qemu_host_page_size = TARGET_PAGE_SIZE;
268 qemu_host_page_bits = 0;
269 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
270 qemu_host_page_bits++;
271 qemu_host_page_mask = ~(qemu_host_page_size - 1);
272 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
273 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
275 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
277 long long startaddr, endaddr;
278 FILE *f;
279 int n;
281 mmap_lock();
282 last_brk = (unsigned long)sbrk(0);
283 f = fopen("/proc/self/maps", "r");
284 if (f) {
285 do {
286 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
287 if (n == 2) {
288 startaddr = MIN(startaddr,
289 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
290 endaddr = MIN(endaddr,
291 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
292 page_set_flags(startaddr & TARGET_PAGE_MASK,
293 TARGET_PAGE_ALIGN(endaddr),
294 PAGE_RESERVED);
296 } while (!feof(f));
297 fclose(f);
299 mmap_unlock();
301 #endif
304 static inline PageDesc **page_l1_map(target_ulong index)
306 #if TARGET_LONG_BITS > 32
307 /* Host memory outside guest VM. For 32-bit targets we have already
308 excluded high addresses. */
309 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
310 return NULL;
311 #endif
312 return &l1_map[index >> L2_BITS];
315 static inline PageDesc *page_find_alloc(target_ulong index)
317 PageDesc **lp, *p;
318 lp = page_l1_map(index);
319 if (!lp)
320 return NULL;
322 p = *lp;
323 if (!p) {
324 /* allocate if not found */
325 #if defined(CONFIG_USER_ONLY)
326 size_t len = sizeof(PageDesc) * L2_SIZE;
327 /* Don't use qemu_malloc because it may recurse. */
328 p = mmap(NULL, len, PROT_READ | PROT_WRITE,
329 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
330 *lp = p;
331 if (h2g_valid(p)) {
332 unsigned long addr = h2g(p);
333 page_set_flags(addr & TARGET_PAGE_MASK,
334 TARGET_PAGE_ALIGN(addr + len),
335 PAGE_RESERVED);
337 #else
338 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
339 *lp = p;
340 #endif
342 return p + (index & (L2_SIZE - 1));
345 static inline PageDesc *page_find(target_ulong index)
347 PageDesc **lp, *p;
348 lp = page_l1_map(index);
349 if (!lp)
350 return NULL;
352 p = *lp;
353 if (!p) {
354 return NULL;
356 return p + (index & (L2_SIZE - 1));
359 static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
361 void **lp, **p;
362 PhysPageDesc *pd;
364 p = (void **)l1_phys_map;
365 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
367 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
368 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
369 #endif
370 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
371 p = *lp;
372 if (!p) {
373 /* allocate if not found */
374 if (!alloc)
375 return NULL;
376 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
377 memset(p, 0, sizeof(void *) * L1_SIZE);
378 *lp = p;
380 #endif
381 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
382 pd = *lp;
383 if (!pd) {
384 int i;
385 /* allocate if not found */
386 if (!alloc)
387 return NULL;
388 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
389 *lp = pd;
390 for (i = 0; i < L2_SIZE; i++) {
391 pd[i].phys_offset = IO_MEM_UNASSIGNED;
392 pd[i].region_offset = (index + i) << TARGET_PAGE_BITS;
395 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
398 static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
400 return phys_page_find_alloc(index, 0);
403 #if !defined(CONFIG_USER_ONLY)
404 static void tlb_protect_code(ram_addr_t ram_addr);
405 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
406 target_ulong vaddr);
407 #define mmap_lock() do { } while(0)
408 #define mmap_unlock() do { } while(0)
409 #endif
411 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
413 #if defined(CONFIG_USER_ONLY)
414 /* Currently it is not recommended to allocate big chunks of data in
415 user mode. It will change when a dedicated libc will be used */
416 #define USE_STATIC_CODE_GEN_BUFFER
417 #endif
419 #ifdef USE_STATIC_CODE_GEN_BUFFER
420 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
421 #endif
423 static void code_gen_alloc(unsigned long tb_size)
425 if (kvm_enabled())
426 return;
428 #ifdef USE_STATIC_CODE_GEN_BUFFER
429 code_gen_buffer = static_code_gen_buffer;
430 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
431 map_exec(code_gen_buffer, code_gen_buffer_size);
432 #else
433 code_gen_buffer_size = tb_size;
434 if (code_gen_buffer_size == 0) {
435 #if defined(CONFIG_USER_ONLY)
436 /* in user mode, phys_ram_size is not meaningful */
437 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
438 #else
439 /* XXX: needs adjustments */
440 code_gen_buffer_size = (unsigned long)(ram_size / 4);
441 #endif
443 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
444 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
445 /* The code gen buffer location may have constraints depending on
446 the host cpu and OS */
447 #if defined(__linux__)
449 int flags;
450 void *start = NULL;
452 flags = MAP_PRIVATE | MAP_ANONYMOUS;
453 #if defined(__x86_64__)
454 flags |= MAP_32BIT;
455 /* Cannot map more than that */
456 if (code_gen_buffer_size > (800 * 1024 * 1024))
457 code_gen_buffer_size = (800 * 1024 * 1024);
458 #elif defined(__sparc_v9__)
459 // Map the buffer below 2G, so we can use direct calls and branches
460 flags |= MAP_FIXED;
461 start = (void *) 0x60000000UL;
462 if (code_gen_buffer_size > (512 * 1024 * 1024))
463 code_gen_buffer_size = (512 * 1024 * 1024);
464 #elif defined(__arm__)
465 /* Map the buffer below 32M, so we can use direct calls and branches */
466 flags |= MAP_FIXED;
467 start = (void *) 0x01000000UL;
468 if (code_gen_buffer_size > 16 * 1024 * 1024)
469 code_gen_buffer_size = 16 * 1024 * 1024;
470 #endif
471 code_gen_buffer = mmap(start, code_gen_buffer_size,
472 PROT_WRITE | PROT_READ | PROT_EXEC,
473 flags, -1, 0);
474 if (code_gen_buffer == MAP_FAILED) {
475 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
476 exit(1);
479 #elif defined(__FreeBSD__) || defined(__DragonFly__)
481 int flags;
482 void *addr = NULL;
483 flags = MAP_PRIVATE | MAP_ANONYMOUS;
484 #if defined(__x86_64__)
485 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
486 * 0x40000000 is free */
487 flags |= MAP_FIXED;
488 addr = (void *)0x40000000;
489 /* Cannot map more than that */
490 if (code_gen_buffer_size > (800 * 1024 * 1024))
491 code_gen_buffer_size = (800 * 1024 * 1024);
492 #endif
493 code_gen_buffer = mmap(addr, code_gen_buffer_size,
494 PROT_WRITE | PROT_READ | PROT_EXEC,
495 flags, -1, 0);
496 if (code_gen_buffer == MAP_FAILED) {
497 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
498 exit(1);
501 #else
502 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
503 map_exec(code_gen_buffer, code_gen_buffer_size);
504 #endif
505 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
506 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
507 code_gen_buffer_max_size = code_gen_buffer_size -
508 code_gen_max_block_size();
509 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
510 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
513 /* Must be called before using the QEMU cpus. 'tb_size' is the size
514 (in bytes) allocated to the translation buffer. Zero means default
515 size. */
516 void cpu_exec_init_all(unsigned long tb_size)
518 cpu_gen_init();
519 code_gen_alloc(tb_size);
520 code_gen_ptr = code_gen_buffer;
521 page_init();
522 #if !defined(CONFIG_USER_ONLY)
523 io_mem_init();
524 #endif
527 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
529 #define CPU_COMMON_SAVE_VERSION 1
531 static void cpu_common_save(QEMUFile *f, void *opaque)
533 CPUState *env = opaque;
535 cpu_synchronize_state(env, 0);
537 qemu_put_be32s(f, &env->halted);
538 qemu_put_be32s(f, &env->interrupt_request);
541 static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
543 CPUState *env = opaque;
545 if (version_id != CPU_COMMON_SAVE_VERSION)
546 return -EINVAL;
548 qemu_get_be32s(f, &env->halted);
549 qemu_get_be32s(f, &env->interrupt_request);
550 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
551 version_id is increased. */
552 env->interrupt_request &= ~0x01;
553 tlb_flush(env, 1);
554 cpu_synchronize_state(env, 1);
556 return 0;
558 #endif
560 CPUState *qemu_get_cpu(int cpu)
562 CPUState *env = first_cpu;
564 while (env) {
565 if (env->cpu_index == cpu)
566 break;
567 env = env->next_cpu;
570 return env;
573 void cpu_exec_init(CPUState *env)
575 CPUState **penv;
576 int cpu_index;
578 #if defined(CONFIG_USER_ONLY)
579 cpu_list_lock();
580 #endif
581 env->next_cpu = NULL;
582 penv = &first_cpu;
583 cpu_index = 0;
584 while (*penv != NULL) {
585 penv = &(*penv)->next_cpu;
586 cpu_index++;
588 env->cpu_index = cpu_index;
589 env->numa_node = 0;
590 TAILQ_INIT(&env->breakpoints);
591 TAILQ_INIT(&env->watchpoints);
592 #ifdef __WIN32
593 env->thread_id = GetCurrentProcessId();
594 #else
595 env->thread_id = getpid();
596 #endif
597 *penv = env;
598 #if defined(CONFIG_USER_ONLY)
599 cpu_list_unlock();
600 #endif
601 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
602 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
603 cpu_common_save, cpu_common_load, env);
604 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
605 cpu_save, cpu_load, env);
606 #endif
609 static inline void invalidate_page_bitmap(PageDesc *p)
611 if (p->code_bitmap) {
612 qemu_free(p->code_bitmap);
613 p->code_bitmap = NULL;
615 p->code_write_count = 0;
618 /* set to NULL all the 'first_tb' fields in all PageDescs */
619 static void page_flush_tb(void)
621 int i, j;
622 PageDesc *p;
624 for(i = 0; i < L1_SIZE; i++) {
625 p = l1_map[i];
626 if (p) {
627 for(j = 0; j < L2_SIZE; j++) {
628 p->first_tb = NULL;
629 invalidate_page_bitmap(p);
630 p++;
636 /* flush all the translation blocks */
637 /* XXX: tb_flush is currently not thread safe */
638 void tb_flush(CPUState *env1)
640 CPUState *env;
641 #if defined(DEBUG_FLUSH)
642 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
643 (unsigned long)(code_gen_ptr - code_gen_buffer),
644 nb_tbs, nb_tbs > 0 ?
645 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
646 #endif
647 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
648 cpu_abort(env1, "Internal error: code buffer overflow\n");
650 nb_tbs = 0;
652 for(env = first_cpu; env != NULL; env = env->next_cpu) {
653 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
656 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
657 page_flush_tb();
659 code_gen_ptr = code_gen_buffer;
660 /* XXX: flush processor icache at this point if cache flush is
661 expensive */
662 tb_flush_count++;
665 #ifdef DEBUG_TB_CHECK
667 static void tb_invalidate_check(target_ulong address)
669 TranslationBlock *tb;
670 int i;
671 address &= TARGET_PAGE_MASK;
672 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
673 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
674 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
675 address >= tb->pc + tb->size)) {
676 printf("ERROR invalidate: address=" TARGET_FMT_lx
677 " PC=%08lx size=%04x\n",
678 address, (long)tb->pc, tb->size);
684 /* verify that all the pages have correct rights for code */
685 static void tb_page_check(void)
687 TranslationBlock *tb;
688 int i, flags1, flags2;
690 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
691 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
692 flags1 = page_get_flags(tb->pc);
693 flags2 = page_get_flags(tb->pc + tb->size - 1);
694 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
695 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
696 (long)tb->pc, tb->size, flags1, flags2);
702 #endif
704 /* invalidate one TB */
705 static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
706 int next_offset)
708 TranslationBlock *tb1;
709 for(;;) {
710 tb1 = *ptb;
711 if (tb1 == tb) {
712 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
713 break;
715 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
719 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
721 TranslationBlock *tb1;
722 unsigned int n1;
724 for(;;) {
725 tb1 = *ptb;
726 n1 = (long)tb1 & 3;
727 tb1 = (TranslationBlock *)((long)tb1 & ~3);
728 if (tb1 == tb) {
729 *ptb = tb1->page_next[n1];
730 break;
732 ptb = &tb1->page_next[n1];
736 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
738 TranslationBlock *tb1, **ptb;
739 unsigned int n1;
741 ptb = &tb->jmp_next[n];
742 tb1 = *ptb;
743 if (tb1) {
744 /* find tb(n) in circular list */
745 for(;;) {
746 tb1 = *ptb;
747 n1 = (long)tb1 & 3;
748 tb1 = (TranslationBlock *)((long)tb1 & ~3);
749 if (n1 == n && tb1 == tb)
750 break;
751 if (n1 == 2) {
752 ptb = &tb1->jmp_first;
753 } else {
754 ptb = &tb1->jmp_next[n1];
757 /* now we can suppress tb(n) from the list */
758 *ptb = tb->jmp_next[n];
760 tb->jmp_next[n] = NULL;
764 /* reset the jump entry 'n' of a TB so that it is not chained to
765 another TB */
766 static inline void tb_reset_jump(TranslationBlock *tb, int n)
768 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
771 void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
773 CPUState *env;
774 PageDesc *p;
775 unsigned int h, n1;
776 target_phys_addr_t phys_pc;
777 TranslationBlock *tb1, *tb2;
779 /* remove the TB from the hash list */
780 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
781 h = tb_phys_hash_func(phys_pc);
782 tb_remove(&tb_phys_hash[h], tb,
783 offsetof(TranslationBlock, phys_hash_next));
785 /* remove the TB from the page list */
786 if (tb->page_addr[0] != page_addr) {
787 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
788 tb_page_remove(&p->first_tb, tb);
789 invalidate_page_bitmap(p);
791 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
792 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
793 tb_page_remove(&p->first_tb, tb);
794 invalidate_page_bitmap(p);
797 tb_invalidated_flag = 1;
799 /* remove the TB from the hash list */
800 h = tb_jmp_cache_hash_func(tb->pc);
801 for(env = first_cpu; env != NULL; env = env->next_cpu) {
802 if (env->tb_jmp_cache[h] == tb)
803 env->tb_jmp_cache[h] = NULL;
806 /* suppress this TB from the two jump lists */
807 tb_jmp_remove(tb, 0);
808 tb_jmp_remove(tb, 1);
810 /* suppress any remaining jumps to this TB */
811 tb1 = tb->jmp_first;
812 for(;;) {
813 n1 = (long)tb1 & 3;
814 if (n1 == 2)
815 break;
816 tb1 = (TranslationBlock *)((long)tb1 & ~3);
817 tb2 = tb1->jmp_next[n1];
818 tb_reset_jump(tb1, n1);
819 tb1->jmp_next[n1] = NULL;
820 tb1 = tb2;
822 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
824 tb_phys_invalidate_count++;
827 static inline void set_bits(uint8_t *tab, int start, int len)
829 int end, mask, end1;
831 end = start + len;
832 tab += start >> 3;
833 mask = 0xff << (start & 7);
834 if ((start & ~7) == (end & ~7)) {
835 if (start < end) {
836 mask &= ~(0xff << (end & 7));
837 *tab |= mask;
839 } else {
840 *tab++ |= mask;
841 start = (start + 8) & ~7;
842 end1 = end & ~7;
843 while (start < end1) {
844 *tab++ = 0xff;
845 start += 8;
847 if (start < end) {
848 mask = ~(0xff << (end & 7));
849 *tab |= mask;
854 static void build_page_bitmap(PageDesc *p)
856 int n, tb_start, tb_end;
857 TranslationBlock *tb;
859 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
861 tb = p->first_tb;
862 while (tb != NULL) {
863 n = (long)tb & 3;
864 tb = (TranslationBlock *)((long)tb & ~3);
865 /* NOTE: this is subtle as a TB may span two physical pages */
866 if (n == 0) {
867 /* NOTE: tb_end may be after the end of the page, but
868 it is not a problem */
869 tb_start = tb->pc & ~TARGET_PAGE_MASK;
870 tb_end = tb_start + tb->size;
871 if (tb_end > TARGET_PAGE_SIZE)
872 tb_end = TARGET_PAGE_SIZE;
873 } else {
874 tb_start = 0;
875 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
877 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
878 tb = tb->page_next[n];
882 TranslationBlock *tb_gen_code(CPUState *env,
883 target_ulong pc, target_ulong cs_base,
884 int flags, int cflags)
886 TranslationBlock *tb;
887 uint8_t *tc_ptr;
888 target_ulong phys_pc, phys_page2, virt_page2;
889 int code_gen_size;
891 phys_pc = get_phys_addr_code(env, pc);
892 tb = tb_alloc(pc);
893 if (!tb) {
894 /* flush must be done */
895 tb_flush(env);
896 /* cannot fail at this point */
897 tb = tb_alloc(pc);
898 /* Don't forget to invalidate previous TB info. */
899 tb_invalidated_flag = 1;
901 tc_ptr = code_gen_ptr;
902 tb->tc_ptr = tc_ptr;
903 tb->cs_base = cs_base;
904 tb->flags = flags;
905 tb->cflags = cflags;
906 cpu_gen_code(env, tb, &code_gen_size);
907 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
909 /* check next page if needed */
910 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
911 phys_page2 = -1;
912 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
913 phys_page2 = get_phys_addr_code(env, virt_page2);
915 tb_link_phys(tb, phys_pc, phys_page2);
916 return tb;
919 /* invalidate all TBs which intersect with the target physical page
920 starting in range [start;end[. NOTE: start and end must refer to
921 the same physical page. 'is_cpu_write_access' should be true if called
922 from a real cpu write access: the virtual CPU will exit the current
923 TB if code is modified inside this TB. */
924 void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
925 int is_cpu_write_access)
927 TranslationBlock *tb, *tb_next, *saved_tb;
928 CPUState *env = cpu_single_env;
929 target_ulong tb_start, tb_end;
930 PageDesc *p;
931 int n;
932 #ifdef TARGET_HAS_PRECISE_SMC
933 int current_tb_not_found = is_cpu_write_access;
934 TranslationBlock *current_tb = NULL;
935 int current_tb_modified = 0;
936 target_ulong current_pc = 0;
937 target_ulong current_cs_base = 0;
938 int current_flags = 0;
939 #endif /* TARGET_HAS_PRECISE_SMC */
941 p = page_find(start >> TARGET_PAGE_BITS);
942 if (!p)
943 return;
944 if (!p->code_bitmap &&
945 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
946 is_cpu_write_access) {
947 /* build code bitmap */
948 build_page_bitmap(p);
951 /* we remove all the TBs in the range [start, end[ */
952 /* XXX: see if in some cases it could be faster to invalidate all the code */
953 tb = p->first_tb;
954 while (tb != NULL) {
955 n = (long)tb & 3;
956 tb = (TranslationBlock *)((long)tb & ~3);
957 tb_next = tb->page_next[n];
958 /* NOTE: this is subtle as a TB may span two physical pages */
959 if (n == 0) {
960 /* NOTE: tb_end may be after the end of the page, but
961 it is not a problem */
962 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
963 tb_end = tb_start + tb->size;
964 } else {
965 tb_start = tb->page_addr[1];
966 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
968 if (!(tb_end <= start || tb_start >= end)) {
969 #ifdef TARGET_HAS_PRECISE_SMC
970 if (current_tb_not_found) {
971 current_tb_not_found = 0;
972 current_tb = NULL;
973 if (env->mem_io_pc) {
974 /* now we have a real cpu fault */
975 current_tb = tb_find_pc(env->mem_io_pc);
978 if (current_tb == tb &&
979 (current_tb->cflags & CF_COUNT_MASK) != 1) {
980 /* If we are modifying the current TB, we must stop
981 its execution. We could be more precise by checking
982 that the modification is after the current PC, but it
983 would require a specialized function to partially
984 restore the CPU state */
986 current_tb_modified = 1;
987 cpu_restore_state(current_tb, env,
988 env->mem_io_pc, NULL);
989 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
990 &current_flags);
992 #endif /* TARGET_HAS_PRECISE_SMC */
993 /* we need to do that to handle the case where a signal
994 occurs while doing tb_phys_invalidate() */
995 saved_tb = NULL;
996 if (env) {
997 saved_tb = env->current_tb;
998 env->current_tb = NULL;
1000 tb_phys_invalidate(tb, -1);
1001 if (env) {
1002 env->current_tb = saved_tb;
1003 if (env->interrupt_request && env->current_tb)
1004 cpu_interrupt(env, env->interrupt_request);
1007 tb = tb_next;
1009 #if !defined(CONFIG_USER_ONLY)
1010 /* if no code remaining, no need to continue to use slow writes */
1011 if (!p->first_tb) {
1012 invalidate_page_bitmap(p);
1013 if (is_cpu_write_access) {
1014 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1017 #endif
1018 #ifdef TARGET_HAS_PRECISE_SMC
1019 if (current_tb_modified) {
1020 /* we generate a block containing just the instruction
1021 modifying the memory. It will ensure that it cannot modify
1022 itself */
1023 env->current_tb = NULL;
1024 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1025 cpu_resume_from_signal(env, NULL);
1027 #endif
1030 /* len must be <= 8 and start must be a multiple of len */
1031 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
1033 PageDesc *p;
1034 int offset, b;
1035 #if 0
1036 if (1) {
1037 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1038 cpu_single_env->mem_io_vaddr, len,
1039 cpu_single_env->eip,
1040 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
1042 #endif
1043 p = page_find(start >> TARGET_PAGE_BITS);
1044 if (!p)
1045 return;
1046 if (p->code_bitmap) {
1047 offset = start & ~TARGET_PAGE_MASK;
1048 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1049 if (b & ((1 << len) - 1))
1050 goto do_invalidate;
1051 } else {
1052 do_invalidate:
1053 tb_invalidate_phys_page_range(start, start + len, 1);
1057 #if !defined(CONFIG_SOFTMMU)
1058 static void tb_invalidate_phys_page(target_phys_addr_t addr,
1059 unsigned long pc, void *puc)
1061 TranslationBlock *tb;
1062 PageDesc *p;
1063 int n;
1064 #ifdef TARGET_HAS_PRECISE_SMC
1065 TranslationBlock *current_tb = NULL;
1066 CPUState *env = cpu_single_env;
1067 int current_tb_modified = 0;
1068 target_ulong current_pc = 0;
1069 target_ulong current_cs_base = 0;
1070 int current_flags = 0;
1071 #endif
1073 addr &= TARGET_PAGE_MASK;
1074 p = page_find(addr >> TARGET_PAGE_BITS);
1075 if (!p)
1076 return;
1077 tb = p->first_tb;
1078 #ifdef TARGET_HAS_PRECISE_SMC
1079 if (tb && pc != 0) {
1080 current_tb = tb_find_pc(pc);
1082 #endif
1083 while (tb != NULL) {
1084 n = (long)tb & 3;
1085 tb = (TranslationBlock *)((long)tb & ~3);
1086 #ifdef TARGET_HAS_PRECISE_SMC
1087 if (current_tb == tb &&
1088 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1089 /* If we are modifying the current TB, we must stop
1090 its execution. We could be more precise by checking
1091 that the modification is after the current PC, but it
1092 would require a specialized function to partially
1093 restore the CPU state */
1095 current_tb_modified = 1;
1096 cpu_restore_state(current_tb, env, pc, puc);
1097 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1098 &current_flags);
1100 #endif /* TARGET_HAS_PRECISE_SMC */
1101 tb_phys_invalidate(tb, addr);
1102 tb = tb->page_next[n];
1104 p->first_tb = NULL;
1105 #ifdef TARGET_HAS_PRECISE_SMC
1106 if (current_tb_modified) {
1107 /* we generate a block containing just the instruction
1108 modifying the memory. It will ensure that it cannot modify
1109 itself */
1110 env->current_tb = NULL;
1111 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1112 cpu_resume_from_signal(env, puc);
1114 #endif
1116 #endif
1118 /* add the tb in the target page and protect it if necessary */
1119 static inline void tb_alloc_page(TranslationBlock *tb,
1120 unsigned int n, target_ulong page_addr)
1122 PageDesc *p;
1123 TranslationBlock *last_first_tb;
1125 tb->page_addr[n] = page_addr;
1126 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
1127 tb->page_next[n] = p->first_tb;
1128 last_first_tb = p->first_tb;
1129 p->first_tb = (TranslationBlock *)((long)tb | n);
1130 invalidate_page_bitmap(p);
1132 #if defined(TARGET_HAS_SMC) || 1
1134 #if defined(CONFIG_USER_ONLY)
1135 if (p->flags & PAGE_WRITE) {
1136 target_ulong addr;
1137 PageDesc *p2;
1138 int prot;
1140 /* force the host page as non writable (writes will have a
1141 page fault + mprotect overhead) */
1142 page_addr &= qemu_host_page_mask;
1143 prot = 0;
1144 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1145 addr += TARGET_PAGE_SIZE) {
1147 p2 = page_find (addr >> TARGET_PAGE_BITS);
1148 if (!p2)
1149 continue;
1150 prot |= p2->flags;
1151 p2->flags &= ~PAGE_WRITE;
1152 page_get_flags(addr);
1154 mprotect(g2h(page_addr), qemu_host_page_size,
1155 (prot & PAGE_BITS) & ~PAGE_WRITE);
1156 #ifdef DEBUG_TB_INVALIDATE
1157 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1158 page_addr);
1159 #endif
1161 #else
1162 /* if some code is already present, then the pages are already
1163 protected. So we handle the case where only the first TB is
1164 allocated in a physical page */
1165 if (!last_first_tb) {
1166 tlb_protect_code(page_addr);
1168 #endif
1170 #endif /* TARGET_HAS_SMC */
1173 /* Allocate a new translation block. Flush the translation buffer if
1174 too many translation blocks or too much generated code. */
1175 TranslationBlock *tb_alloc(target_ulong pc)
1177 TranslationBlock *tb;
1179 if (nb_tbs >= code_gen_max_blocks ||
1180 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
1181 return NULL;
1182 tb = &tbs[nb_tbs++];
1183 tb->pc = pc;
1184 tb->cflags = 0;
1185 return tb;
1188 void tb_free(TranslationBlock *tb)
1190 /* In practice this is mostly used for single use temporary TB
1191 Ignore the hard cases and just back up if this TB happens to
1192 be the last one generated. */
1193 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1194 code_gen_ptr = tb->tc_ptr;
1195 nb_tbs--;
1199 /* add a new TB and link it to the physical page tables. phys_page2 is
1200 (-1) to indicate that only one page contains the TB. */
1201 void tb_link_phys(TranslationBlock *tb,
1202 target_ulong phys_pc, target_ulong phys_page2)
1204 unsigned int h;
1205 TranslationBlock **ptb;
1207 /* Grab the mmap lock to stop another thread invalidating this TB
1208 before we are done. */
1209 mmap_lock();
1210 /* add in the physical hash table */
1211 h = tb_phys_hash_func(phys_pc);
1212 ptb = &tb_phys_hash[h];
1213 tb->phys_hash_next = *ptb;
1214 *ptb = tb;
1216 /* add in the page list */
1217 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1218 if (phys_page2 != -1)
1219 tb_alloc_page(tb, 1, phys_page2);
1220 else
1221 tb->page_addr[1] = -1;
1223 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1224 tb->jmp_next[0] = NULL;
1225 tb->jmp_next[1] = NULL;
1227 /* init original jump addresses */
1228 if (tb->tb_next_offset[0] != 0xffff)
1229 tb_reset_jump(tb, 0);
1230 if (tb->tb_next_offset[1] != 0xffff)
1231 tb_reset_jump(tb, 1);
1233 #ifdef DEBUG_TB_CHECK
1234 tb_page_check();
1235 #endif
1236 mmap_unlock();
1239 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1240 tb[1].tc_ptr. Return NULL if not found */
1241 TranslationBlock *tb_find_pc(unsigned long tc_ptr)
1243 int m_min, m_max, m;
1244 unsigned long v;
1245 TranslationBlock *tb;
1247 if (nb_tbs <= 0)
1248 return NULL;
1249 if (tc_ptr < (unsigned long)code_gen_buffer ||
1250 tc_ptr >= (unsigned long)code_gen_ptr)
1251 return NULL;
1252 /* binary search (cf Knuth) */
1253 m_min = 0;
1254 m_max = nb_tbs - 1;
1255 while (m_min <= m_max) {
1256 m = (m_min + m_max) >> 1;
1257 tb = &tbs[m];
1258 v = (unsigned long)tb->tc_ptr;
1259 if (v == tc_ptr)
1260 return tb;
1261 else if (tc_ptr < v) {
1262 m_max = m - 1;
1263 } else {
1264 m_min = m + 1;
1267 return &tbs[m_max];
1270 static void tb_reset_jump_recursive(TranslationBlock *tb);
1272 static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1274 TranslationBlock *tb1, *tb_next, **ptb;
1275 unsigned int n1;
1277 tb1 = tb->jmp_next[n];
1278 if (tb1 != NULL) {
1279 /* find head of list */
1280 for(;;) {
1281 n1 = (long)tb1 & 3;
1282 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1283 if (n1 == 2)
1284 break;
1285 tb1 = tb1->jmp_next[n1];
1287 /* we are now sure now that tb jumps to tb1 */
1288 tb_next = tb1;
1290 /* remove tb from the jmp_first list */
1291 ptb = &tb_next->jmp_first;
1292 for(;;) {
1293 tb1 = *ptb;
1294 n1 = (long)tb1 & 3;
1295 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1296 if (n1 == n && tb1 == tb)
1297 break;
1298 ptb = &tb1->jmp_next[n1];
1300 *ptb = tb->jmp_next[n];
1301 tb->jmp_next[n] = NULL;
1303 /* suppress the jump to next tb in generated code */
1304 tb_reset_jump(tb, n);
1306 /* suppress jumps in the tb on which we could have jumped */
1307 tb_reset_jump_recursive(tb_next);
1311 static void tb_reset_jump_recursive(TranslationBlock *tb)
1313 tb_reset_jump_recursive2(tb, 0);
1314 tb_reset_jump_recursive2(tb, 1);
1317 #if defined(TARGET_HAS_ICE)
1318 static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1320 target_phys_addr_t addr;
1321 target_ulong pd;
1322 ram_addr_t ram_addr;
1323 PhysPageDesc *p;
1325 addr = cpu_get_phys_page_debug(env, pc);
1326 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1327 if (!p) {
1328 pd = IO_MEM_UNASSIGNED;
1329 } else {
1330 pd = p->phys_offset;
1332 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
1333 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1335 #endif
1337 /* Add a watchpoint. */
1338 int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1339 int flags, CPUWatchpoint **watchpoint)
1341 target_ulong len_mask = ~(len - 1);
1342 CPUWatchpoint *wp;
1344 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1345 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1346 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1347 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1348 return -EINVAL;
1350 wp = qemu_malloc(sizeof(*wp));
1352 wp->vaddr = addr;
1353 wp->len_mask = len_mask;
1354 wp->flags = flags;
1356 /* keep all GDB-injected watchpoints in front */
1357 if (flags & BP_GDB)
1358 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1359 else
1360 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
1362 tlb_flush_page(env, addr);
1364 if (watchpoint)
1365 *watchpoint = wp;
1366 return 0;
1369 /* Remove a specific watchpoint. */
1370 int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1371 int flags)
1373 target_ulong len_mask = ~(len - 1);
1374 CPUWatchpoint *wp;
1376 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1377 if (addr == wp->vaddr && len_mask == wp->len_mask
1378 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
1379 cpu_watchpoint_remove_by_ref(env, wp);
1380 return 0;
1383 return -ENOENT;
1386 /* Remove a specific watchpoint by reference. */
1387 void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1389 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
1391 tlb_flush_page(env, watchpoint->vaddr);
1393 qemu_free(watchpoint);
1396 /* Remove all matching watchpoints. */
1397 void cpu_watchpoint_remove_all(CPUState *env, int mask)
1399 CPUWatchpoint *wp, *next;
1401 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
1402 if (wp->flags & mask)
1403 cpu_watchpoint_remove_by_ref(env, wp);
1407 /* Add a breakpoint. */
1408 int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1409 CPUBreakpoint **breakpoint)
1411 #if defined(TARGET_HAS_ICE)
1412 CPUBreakpoint *bp;
1414 bp = qemu_malloc(sizeof(*bp));
1416 bp->pc = pc;
1417 bp->flags = flags;
1419 /* keep all GDB-injected breakpoints in front */
1420 if (flags & BP_GDB)
1421 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1422 else
1423 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
1425 breakpoint_invalidate(env, pc);
1427 if (breakpoint)
1428 *breakpoint = bp;
1429 return 0;
1430 #else
1431 return -ENOSYS;
1432 #endif
1435 /* Remove a specific breakpoint. */
1436 int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1438 #if defined(TARGET_HAS_ICE)
1439 CPUBreakpoint *bp;
1441 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1442 if (bp->pc == pc && bp->flags == flags) {
1443 cpu_breakpoint_remove_by_ref(env, bp);
1444 return 0;
1447 return -ENOENT;
1448 #else
1449 return -ENOSYS;
1450 #endif
1453 /* Remove a specific breakpoint by reference. */
1454 void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
1456 #if defined(TARGET_HAS_ICE)
1457 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
1459 breakpoint_invalidate(env, breakpoint->pc);
1461 qemu_free(breakpoint);
1462 #endif
1465 /* Remove all matching breakpoints. */
1466 void cpu_breakpoint_remove_all(CPUState *env, int mask)
1468 #if defined(TARGET_HAS_ICE)
1469 CPUBreakpoint *bp, *next;
1471 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
1472 if (bp->flags & mask)
1473 cpu_breakpoint_remove_by_ref(env, bp);
1475 #endif
1478 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1479 CPU loop after each instruction */
1480 void cpu_single_step(CPUState *env, int enabled)
1482 #if defined(TARGET_HAS_ICE)
1483 if (env->singlestep_enabled != enabled) {
1484 env->singlestep_enabled = enabled;
1485 if (kvm_enabled())
1486 kvm_update_guest_debug(env, 0);
1487 else {
1488 /* must flush all the translated code to avoid inconsistencies */
1489 /* XXX: only flush what is necessary */
1490 tb_flush(env);
1493 #endif
1496 /* enable or disable low levels log */
1497 void cpu_set_log(int log_flags)
1499 loglevel = log_flags;
1500 if (loglevel && !logfile) {
1501 logfile = fopen(logfilename, log_append ? "a" : "w");
1502 if (!logfile) {
1503 perror(logfilename);
1504 _exit(1);
1506 #if !defined(CONFIG_SOFTMMU)
1507 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1509 static char logfile_buf[4096];
1510 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1512 #else
1513 setvbuf(logfile, NULL, _IOLBF, 0);
1514 #endif
1515 log_append = 1;
1517 if (!loglevel && logfile) {
1518 fclose(logfile);
1519 logfile = NULL;
1523 void cpu_set_log_filename(const char *filename)
1525 logfilename = strdup(filename);
1526 if (logfile) {
1527 fclose(logfile);
1528 logfile = NULL;
1530 cpu_set_log(loglevel);
1533 static void cpu_unlink_tb(CPUState *env)
1535 #if defined(USE_NPTL)
1536 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1537 problem and hope the cpu will stop of its own accord. For userspace
1538 emulation this often isn't actually as bad as it sounds. Often
1539 signals are used primarily to interrupt blocking syscalls. */
1540 #else
1541 TranslationBlock *tb;
1542 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
1544 tb = env->current_tb;
1545 /* if the cpu is currently executing code, we must unlink it and
1546 all the potentially executing TB */
1547 if (tb && !testandset(&interrupt_lock)) {
1548 env->current_tb = NULL;
1549 tb_reset_jump_recursive(tb);
1550 resetlock(&interrupt_lock);
1552 #endif
1555 /* mask must never be zero, except for A20 change call */
1556 void cpu_interrupt(CPUState *env, int mask)
1558 int old_mask;
1560 old_mask = env->interrupt_request;
1561 env->interrupt_request |= mask;
1562 if (kvm_enabled() && !qemu_kvm_irqchip_in_kernel())
1563 kvm_update_interrupt_request(env);
1565 #ifndef CONFIG_USER_ONLY
1567 * If called from iothread context, wake the target cpu in
1568 * case its halted.
1570 if (!qemu_cpu_self(env)) {
1571 qemu_cpu_kick(env);
1572 return;
1574 #endif
1576 if (use_icount) {
1577 env->icount_decr.u16.high = 0xffff;
1578 #ifndef CONFIG_USER_ONLY
1579 if (!can_do_io(env)
1580 && (mask & ~old_mask) != 0) {
1581 cpu_abort(env, "Raised interrupt while not in I/O function");
1583 #endif
1584 } else {
1585 cpu_unlink_tb(env);
1589 void cpu_reset_interrupt(CPUState *env, int mask)
1591 env->interrupt_request &= ~mask;
1594 void cpu_exit(CPUState *env)
1596 env->exit_request = 1;
1597 cpu_unlink_tb(env);
1600 const CPULogItem cpu_log_items[] = {
1601 { CPU_LOG_TB_OUT_ASM, "out_asm",
1602 "show generated host assembly code for each compiled TB" },
1603 { CPU_LOG_TB_IN_ASM, "in_asm",
1604 "show target assembly code for each compiled TB" },
1605 { CPU_LOG_TB_OP, "op",
1606 "show micro ops for each compiled TB" },
1607 { CPU_LOG_TB_OP_OPT, "op_opt",
1608 "show micro ops "
1609 #ifdef TARGET_I386
1610 "before eflags optimization and "
1611 #endif
1612 "after liveness analysis" },
1613 { CPU_LOG_INT, "int",
1614 "show interrupts/exceptions in short format" },
1615 { CPU_LOG_EXEC, "exec",
1616 "show trace before each executed TB (lots of logs)" },
1617 { CPU_LOG_TB_CPU, "cpu",
1618 "show CPU state before block translation" },
1619 #ifdef TARGET_I386
1620 { CPU_LOG_PCALL, "pcall",
1621 "show protected mode far calls/returns/exceptions" },
1622 { CPU_LOG_RESET, "cpu_reset",
1623 "show CPU state before CPU resets" },
1624 #endif
1625 #ifdef DEBUG_IOPORT
1626 { CPU_LOG_IOPORT, "ioport",
1627 "show all i/o ports accesses" },
1628 #endif
1629 { 0, NULL, NULL },
1632 static int cmp1(const char *s1, int n, const char *s2)
1634 if (strlen(s2) != n)
1635 return 0;
1636 return memcmp(s1, s2, n) == 0;
1639 /* takes a comma separated list of log masks. Return 0 if error. */
1640 int cpu_str_to_log_mask(const char *str)
1642 const CPULogItem *item;
1643 int mask;
1644 const char *p, *p1;
1646 p = str;
1647 mask = 0;
1648 for(;;) {
1649 p1 = strchr(p, ',');
1650 if (!p1)
1651 p1 = p + strlen(p);
1652 if(cmp1(p,p1-p,"all")) {
1653 for(item = cpu_log_items; item->mask != 0; item++) {
1654 mask |= item->mask;
1656 } else {
1657 for(item = cpu_log_items; item->mask != 0; item++) {
1658 if (cmp1(p, p1 - p, item->name))
1659 goto found;
1661 return 0;
1663 found:
1664 mask |= item->mask;
1665 if (*p1 != ',')
1666 break;
1667 p = p1 + 1;
1669 return mask;
1672 void cpu_abort(CPUState *env, const char *fmt, ...)
1674 va_list ap;
1675 va_list ap2;
1677 va_start(ap, fmt);
1678 va_copy(ap2, ap);
1679 fprintf(stderr, "qemu: fatal: ");
1680 vfprintf(stderr, fmt, ap);
1681 fprintf(stderr, "\n");
1682 #ifdef TARGET_I386
1683 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1684 #else
1685 cpu_dump_state(env, stderr, fprintf, 0);
1686 #endif
1687 if (qemu_log_enabled()) {
1688 qemu_log("qemu: fatal: ");
1689 qemu_log_vprintf(fmt, ap2);
1690 qemu_log("\n");
1691 #ifdef TARGET_I386
1692 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
1693 #else
1694 log_cpu_state(env, 0);
1695 #endif
1696 qemu_log_flush();
1697 qemu_log_close();
1699 va_end(ap2);
1700 va_end(ap);
1701 abort();
1704 CPUState *cpu_copy(CPUState *env)
1706 CPUState *new_env = cpu_init(env->cpu_model_str);
1707 CPUState *next_cpu = new_env->next_cpu;
1708 int cpu_index = new_env->cpu_index;
1709 #if defined(TARGET_HAS_ICE)
1710 CPUBreakpoint *bp;
1711 CPUWatchpoint *wp;
1712 #endif
1714 memcpy(new_env, env, sizeof(CPUState));
1716 /* Preserve chaining and index. */
1717 new_env->next_cpu = next_cpu;
1718 new_env->cpu_index = cpu_index;
1720 /* Clone all break/watchpoints.
1721 Note: Once we support ptrace with hw-debug register access, make sure
1722 BP_CPU break/watchpoints are handled correctly on clone. */
1723 TAILQ_INIT(&env->breakpoints);
1724 TAILQ_INIT(&env->watchpoints);
1725 #if defined(TARGET_HAS_ICE)
1726 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1727 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1729 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1730 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1731 wp->flags, NULL);
1733 #endif
1735 return new_env;
1738 #if !defined(CONFIG_USER_ONLY)
1740 static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1742 unsigned int i;
1744 /* Discard jump cache entries for any tb which might potentially
1745 overlap the flushed page. */
1746 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1747 memset (&env->tb_jmp_cache[i], 0,
1748 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1750 i = tb_jmp_cache_hash_page(addr);
1751 memset (&env->tb_jmp_cache[i], 0,
1752 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1755 static CPUTLBEntry s_cputlb_empty_entry = {
1756 .addr_read = -1,
1757 .addr_write = -1,
1758 .addr_code = -1,
1759 .addend = -1,
1762 /* NOTE: if flush_global is true, also flush global entries (not
1763 implemented yet) */
1764 void tlb_flush(CPUState *env, int flush_global)
1766 int i;
1768 #if defined(DEBUG_TLB)
1769 printf("tlb_flush:\n");
1770 #endif
1771 /* must reset current TB so that interrupts cannot modify the
1772 links while we are modifying them */
1773 env->current_tb = NULL;
1775 for(i = 0; i < CPU_TLB_SIZE; i++) {
1776 int mmu_idx;
1777 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1778 env->tlb_table[mmu_idx][i] = s_cputlb_empty_entry;
1782 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
1784 #ifdef CONFIG_KQEMU
1785 if (env->kqemu_enabled) {
1786 kqemu_flush(env, flush_global);
1788 #endif
1789 tlb_flush_count++;
1792 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
1794 if (addr == (tlb_entry->addr_read &
1795 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1796 addr == (tlb_entry->addr_write &
1797 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
1798 addr == (tlb_entry->addr_code &
1799 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1800 *tlb_entry = s_cputlb_empty_entry;
1804 void tlb_flush_page(CPUState *env, target_ulong addr)
1806 int i;
1807 int mmu_idx;
1809 #if defined(DEBUG_TLB)
1810 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
1811 #endif
1812 /* must reset current TB so that interrupts cannot modify the
1813 links while we are modifying them */
1814 env->current_tb = NULL;
1816 addr &= TARGET_PAGE_MASK;
1817 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1818 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1819 tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
1821 tlb_flush_jmp_cache(env, addr);
1823 #ifdef CONFIG_KQEMU
1824 if (env->kqemu_enabled) {
1825 kqemu_flush_page(env, addr);
1827 #endif
1830 /* update the TLBs so that writes to code in the virtual page 'addr'
1831 can be detected */
1832 static void tlb_protect_code(ram_addr_t ram_addr)
1834 cpu_physical_memory_reset_dirty(ram_addr,
1835 ram_addr + TARGET_PAGE_SIZE,
1836 CODE_DIRTY_FLAG);
1839 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1840 tested for self modifying code */
1841 static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
1842 target_ulong vaddr)
1844 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1847 static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1848 unsigned long start, unsigned long length)
1850 unsigned long addr;
1851 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1852 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1853 if ((addr - start) < length) {
1854 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1859 /* Note: start and end must be within the same ram block. */
1860 void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
1861 int dirty_flags)
1863 CPUState *env;
1864 unsigned long length, start1;
1865 int i, mask, len;
1866 uint8_t *p;
1868 start &= TARGET_PAGE_MASK;
1869 end = TARGET_PAGE_ALIGN(end);
1871 length = end - start;
1872 if (length == 0)
1873 return;
1874 len = length >> TARGET_PAGE_BITS;
1875 #ifdef CONFIG_KQEMU
1876 /* XXX: should not depend on cpu context */
1877 env = first_cpu;
1878 if (env->kqemu_enabled) {
1879 ram_addr_t addr;
1880 addr = start;
1881 for(i = 0; i < len; i++) {
1882 kqemu_set_notdirty(env, addr);
1883 addr += TARGET_PAGE_SIZE;
1886 #endif
1887 mask = ~dirty_flags;
1888 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1889 for(i = 0; i < len; i++)
1890 p[i] &= mask;
1892 /* we modify the TLB cache so that the dirty bit will be set again
1893 when accessing the range */
1894 start1 = (unsigned long)qemu_get_ram_ptr(start);
1895 /* Chek that we don't span multiple blocks - this breaks the
1896 address comparisons below. */
1897 if ((unsigned long)qemu_get_ram_ptr(end - 1) - start1
1898 != (end - 1) - start) {
1899 abort();
1902 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1903 int mmu_idx;
1904 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1905 for(i = 0; i < CPU_TLB_SIZE; i++)
1906 tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
1907 start1, length);
1912 int cpu_physical_memory_set_dirty_tracking(int enable)
1914 if (kvm_enabled()) {
1915 return kvm_set_migration_log(enable);
1917 return 0;
1920 int cpu_physical_memory_get_dirty_tracking(void)
1922 return in_migration;
1925 int cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr,
1926 target_phys_addr_t end_addr)
1928 int ret = 0;
1930 if (kvm_enabled())
1931 ret = kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1932 return ret;
1935 static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1937 ram_addr_t ram_addr;
1938 void *p;
1940 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1941 p = (void *)(unsigned long)((tlb_entry->addr_write & TARGET_PAGE_MASK)
1942 + tlb_entry->addend);
1943 ram_addr = qemu_ram_addr_from_host(p);
1944 if (!cpu_physical_memory_is_dirty(ram_addr)) {
1945 tlb_entry->addr_write |= TLB_NOTDIRTY;
1950 /* update the TLB according to the current state of the dirty bits */
1951 void cpu_tlb_update_dirty(CPUState *env)
1953 int i;
1954 int mmu_idx;
1955 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
1956 for(i = 0; i < CPU_TLB_SIZE; i++)
1957 tlb_update_dirty(&env->tlb_table[mmu_idx][i]);
1961 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1963 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1964 tlb_entry->addr_write = vaddr;
1967 /* update the TLB corresponding to virtual page vaddr
1968 so that it is no longer dirty */
1969 static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1971 int i;
1972 int mmu_idx;
1974 vaddr &= TARGET_PAGE_MASK;
1975 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
1976 for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++)
1977 tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
1980 /* add a new TLB entry. At most one entry for a given virtual address
1981 is permitted. Return 0 if OK or 2 if the page could not be mapped
1982 (can only happen in non SOFTMMU mode for I/O pages or pages
1983 conflicting with the host address space). */
1984 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1985 target_phys_addr_t paddr, int prot,
1986 int mmu_idx, int is_softmmu)
1988 PhysPageDesc *p;
1989 unsigned long pd;
1990 unsigned int index;
1991 target_ulong address;
1992 target_ulong code_address;
1993 target_phys_addr_t addend;
1994 int ret;
1995 CPUTLBEntry *te;
1996 CPUWatchpoint *wp;
1997 target_phys_addr_t iotlb;
1999 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
2000 if (!p) {
2001 pd = IO_MEM_UNASSIGNED;
2002 } else {
2003 pd = p->phys_offset;
2005 #if defined(DEBUG_TLB)
2006 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
2007 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
2008 #endif
2010 ret = 0;
2011 address = vaddr;
2012 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
2013 /* IO memory case (romd handled later) */
2014 address |= TLB_MMIO;
2016 addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
2017 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
2018 /* Normal RAM. */
2019 iotlb = pd & TARGET_PAGE_MASK;
2020 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
2021 iotlb |= IO_MEM_NOTDIRTY;
2022 else
2023 iotlb |= IO_MEM_ROM;
2024 } else {
2025 /* IO handlers are currently passed a physical address.
2026 It would be nice to pass an offset from the base address
2027 of that region. This would avoid having to special case RAM,
2028 and avoid full address decoding in every device.
2029 We can't use the high bits of pd for this because
2030 IO_MEM_ROMD uses these as a ram address. */
2031 iotlb = (pd & ~TARGET_PAGE_MASK);
2032 if (p) {
2033 iotlb += p->region_offset;
2034 } else {
2035 iotlb += paddr;
2039 code_address = address;
2040 /* Make accesses to pages with watchpoints go via the
2041 watchpoint trap routines. */
2042 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2043 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
2044 iotlb = io_mem_watch + paddr;
2045 /* TODO: The memory case can be optimized by not trapping
2046 reads of pages with a write breakpoint. */
2047 address |= TLB_MMIO;
2051 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2052 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2053 te = &env->tlb_table[mmu_idx][index];
2054 te->addend = addend - vaddr;
2055 if (prot & PAGE_READ) {
2056 te->addr_read = address;
2057 } else {
2058 te->addr_read = -1;
2061 if (prot & PAGE_EXEC) {
2062 te->addr_code = code_address;
2063 } else {
2064 te->addr_code = -1;
2066 if (prot & PAGE_WRITE) {
2067 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2068 (pd & IO_MEM_ROMD)) {
2069 /* Write access calls the I/O callback. */
2070 te->addr_write = address | TLB_MMIO;
2071 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2072 !cpu_physical_memory_is_dirty(pd)) {
2073 te->addr_write = address | TLB_NOTDIRTY;
2074 } else {
2075 te->addr_write = address;
2077 } else {
2078 te->addr_write = -1;
2080 return ret;
2083 #else
2085 void tlb_flush(CPUState *env, int flush_global)
2089 void tlb_flush_page(CPUState *env, target_ulong addr)
2093 int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2094 target_phys_addr_t paddr, int prot,
2095 int mmu_idx, int is_softmmu)
2097 return 0;
2101 * Walks guest process memory "regions" one by one
2102 * and calls callback function 'fn' for each region.
2104 int walk_memory_regions(void *priv,
2105 int (*fn)(void *, unsigned long, unsigned long, unsigned long))
2107 unsigned long start, end;
2108 PageDesc *p = NULL;
2109 int i, j, prot, prot1;
2110 int rc = 0;
2112 start = end = -1;
2113 prot = 0;
2115 for (i = 0; i <= L1_SIZE; i++) {
2116 p = (i < L1_SIZE) ? l1_map[i] : NULL;
2117 for (j = 0; j < L2_SIZE; j++) {
2118 prot1 = (p == NULL) ? 0 : p[j].flags;
2120 * "region" is one continuous chunk of memory
2121 * that has same protection flags set.
2123 if (prot1 != prot) {
2124 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2125 if (start != -1) {
2126 rc = (*fn)(priv, start, end, prot);
2127 /* callback can stop iteration by returning != 0 */
2128 if (rc != 0)
2129 return (rc);
2131 if (prot1 != 0)
2132 start = end;
2133 else
2134 start = -1;
2135 prot = prot1;
2137 if (p == NULL)
2138 break;
2141 return (rc);
2144 static int dump_region(void *priv, unsigned long start,
2145 unsigned long end, unsigned long prot)
2147 FILE *f = (FILE *)priv;
2149 (void) fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
2150 start, end, end - start,
2151 ((prot & PAGE_READ) ? 'r' : '-'),
2152 ((prot & PAGE_WRITE) ? 'w' : '-'),
2153 ((prot & PAGE_EXEC) ? 'x' : '-'));
2155 return (0);
2158 /* dump memory mappings */
2159 void page_dump(FILE *f)
2161 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
2162 "start", "end", "size", "prot");
2163 walk_memory_regions(f, dump_region);
2166 int page_get_flags(target_ulong address)
2168 PageDesc *p;
2170 p = page_find(address >> TARGET_PAGE_BITS);
2171 if (!p)
2172 return 0;
2173 return p->flags;
2176 /* modify the flags of a page and invalidate the code if
2177 necessary. The flag PAGE_WRITE_ORG is positioned automatically
2178 depending on PAGE_WRITE */
2179 void page_set_flags(target_ulong start, target_ulong end, int flags)
2181 PageDesc *p;
2182 target_ulong addr;
2184 /* mmap_lock should already be held. */
2185 start = start & TARGET_PAGE_MASK;
2186 end = TARGET_PAGE_ALIGN(end);
2187 if (flags & PAGE_WRITE)
2188 flags |= PAGE_WRITE_ORG;
2189 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2190 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
2191 /* We may be called for host regions that are outside guest
2192 address space. */
2193 if (!p)
2194 return;
2195 /* if the write protection is set, then we invalidate the code
2196 inside */
2197 if (!(p->flags & PAGE_WRITE) &&
2198 (flags & PAGE_WRITE) &&
2199 p->first_tb) {
2200 tb_invalidate_phys_page(addr, 0, NULL);
2202 p->flags = flags;
2206 int page_check_range(target_ulong start, target_ulong len, int flags)
2208 PageDesc *p;
2209 target_ulong end;
2210 target_ulong addr;
2212 if (start + len < start)
2213 /* we've wrapped around */
2214 return -1;
2216 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2217 start = start & TARGET_PAGE_MASK;
2219 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2220 p = page_find(addr >> TARGET_PAGE_BITS);
2221 if( !p )
2222 return -1;
2223 if( !(p->flags & PAGE_VALID) )
2224 return -1;
2226 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
2227 return -1;
2228 if (flags & PAGE_WRITE) {
2229 if (!(p->flags & PAGE_WRITE_ORG))
2230 return -1;
2231 /* unprotect the page if it was put read-only because it
2232 contains translated code */
2233 if (!(p->flags & PAGE_WRITE)) {
2234 if (!page_unprotect(addr, 0, NULL))
2235 return -1;
2237 return 0;
2240 return 0;
2243 /* called from signal handler: invalidate the code and unprotect the
2244 page. Return TRUE if the fault was successfully handled. */
2245 int page_unprotect(target_ulong address, unsigned long pc, void *puc)
2247 unsigned int page_index, prot, pindex;
2248 PageDesc *p, *p1;
2249 target_ulong host_start, host_end, addr;
2251 /* Technically this isn't safe inside a signal handler. However we
2252 know this only ever happens in a synchronous SEGV handler, so in
2253 practice it seems to be ok. */
2254 mmap_lock();
2256 host_start = address & qemu_host_page_mask;
2257 page_index = host_start >> TARGET_PAGE_BITS;
2258 p1 = page_find(page_index);
2259 if (!p1) {
2260 mmap_unlock();
2261 return 0;
2263 host_end = host_start + qemu_host_page_size;
2264 p = p1;
2265 prot = 0;
2266 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2267 prot |= p->flags;
2268 p++;
2270 /* if the page was really writable, then we change its
2271 protection back to writable */
2272 if (prot & PAGE_WRITE_ORG) {
2273 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2274 if (!(p1[pindex].flags & PAGE_WRITE)) {
2275 mprotect((void *)g2h(host_start), qemu_host_page_size,
2276 (prot & PAGE_BITS) | PAGE_WRITE);
2277 p1[pindex].flags |= PAGE_WRITE;
2278 /* and since the content will be modified, we must invalidate
2279 the corresponding translated code. */
2280 tb_invalidate_phys_page(address, pc, puc);
2281 #ifdef DEBUG_TB_CHECK
2282 tb_invalidate_check(address);
2283 #endif
2284 mmap_unlock();
2285 return 1;
2288 mmap_unlock();
2289 return 0;
2292 static inline void tlb_set_dirty(CPUState *env,
2293 unsigned long addr, target_ulong vaddr)
2296 #endif /* defined(CONFIG_USER_ONLY) */
2298 #if !defined(CONFIG_USER_ONLY)
2300 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
2301 ram_addr_t memory, ram_addr_t region_offset);
2302 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
2303 ram_addr_t orig_memory, ram_addr_t region_offset);
2304 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2305 need_subpage) \
2306 do { \
2307 if (addr > start_addr) \
2308 start_addr2 = 0; \
2309 else { \
2310 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2311 if (start_addr2 > 0) \
2312 need_subpage = 1; \
2315 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2316 end_addr2 = TARGET_PAGE_SIZE - 1; \
2317 else { \
2318 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2319 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2320 need_subpage = 1; \
2322 } while (0)
2324 /* register physical memory. 'size' must be a multiple of the target
2325 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2326 io memory page. The address used when calling the IO function is
2327 the offset from the start of the region, plus region_offset. Both
2328 start_addr and region_offset are rounded down to a page boundary
2329 before calculating this offset. This should not be a problem unless
2330 the low bits of start_addr and region_offset differ. */
2331 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2332 ram_addr_t size,
2333 ram_addr_t phys_offset,
2334 ram_addr_t region_offset)
2336 target_phys_addr_t addr, end_addr;
2337 PhysPageDesc *p;
2338 CPUState *env;
2339 ram_addr_t orig_size = size;
2340 void *subpage;
2342 #ifdef CONFIG_KQEMU
2343 /* XXX: should not depend on cpu context */
2344 env = first_cpu;
2345 if (env->kqemu_enabled) {
2346 kqemu_set_phys_mem(start_addr, size, phys_offset);
2348 #endif
2349 if (kvm_enabled())
2350 kvm_set_phys_mem(start_addr, size, phys_offset);
2352 if (phys_offset == IO_MEM_UNASSIGNED) {
2353 region_offset = start_addr;
2355 region_offset &= TARGET_PAGE_MASK;
2356 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
2357 end_addr = start_addr + (target_phys_addr_t)size;
2358 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
2359 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2360 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
2361 ram_addr_t orig_memory = p->phys_offset;
2362 target_phys_addr_t start_addr2, end_addr2;
2363 int need_subpage = 0;
2365 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2366 need_subpage);
2367 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2368 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2369 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2370 &p->phys_offset, orig_memory,
2371 p->region_offset);
2372 } else {
2373 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2374 >> IO_MEM_SHIFT];
2376 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2377 region_offset);
2378 p->region_offset = 0;
2379 } else {
2380 p->phys_offset = phys_offset;
2381 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2382 (phys_offset & IO_MEM_ROMD))
2383 phys_offset += TARGET_PAGE_SIZE;
2385 } else {
2386 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2387 p->phys_offset = phys_offset;
2388 p->region_offset = region_offset;
2389 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2390 (phys_offset & IO_MEM_ROMD)) {
2391 phys_offset += TARGET_PAGE_SIZE;
2392 } else {
2393 target_phys_addr_t start_addr2, end_addr2;
2394 int need_subpage = 0;
2396 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2397 end_addr2, need_subpage);
2399 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
2400 subpage = subpage_init((addr & TARGET_PAGE_MASK),
2401 &p->phys_offset, IO_MEM_UNASSIGNED,
2402 addr & TARGET_PAGE_MASK);
2403 subpage_register(subpage, start_addr2, end_addr2,
2404 phys_offset, region_offset);
2405 p->region_offset = 0;
2409 region_offset += TARGET_PAGE_SIZE;
2412 /* since each CPU stores ram addresses in its TLB cache, we must
2413 reset the modified entries */
2414 /* XXX: slow ! */
2415 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2416 tlb_flush(env, 1);
2420 /* XXX: temporary until new memory mapping API */
2421 ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
2423 PhysPageDesc *p;
2425 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2426 if (!p)
2427 return IO_MEM_UNASSIGNED;
2428 return p->phys_offset;
2431 void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2433 if (kvm_enabled())
2434 kvm_coalesce_mmio_region(addr, size);
2437 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2439 if (kvm_enabled())
2440 kvm_uncoalesce_mmio_region(addr, size);
2443 #ifdef CONFIG_KQEMU
2444 /* XXX: better than nothing */
2445 static ram_addr_t kqemu_ram_alloc(ram_addr_t size)
2447 ram_addr_t addr;
2448 if ((last_ram_offset + size) > kqemu_phys_ram_size) {
2449 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
2450 (uint64_t)size, (uint64_t)kqemu_phys_ram_size);
2451 abort();
2453 addr = last_ram_offset;
2454 last_ram_offset = TARGET_PAGE_ALIGN(last_ram_offset + size);
2455 return addr;
2457 #endif
2459 #ifdef __linux__
2461 #include <sys/vfs.h>
2463 #define HUGETLBFS_MAGIC 0x958458f6
2465 static long gethugepagesize(const char *path)
2467 struct statfs fs;
2468 int ret;
2470 do {
2471 ret = statfs(path, &fs);
2472 } while (ret != 0 && errno == EINTR);
2474 if (ret != 0) {
2475 perror("statfs");
2476 return 0;
2479 if (fs.f_type != HUGETLBFS_MAGIC)
2480 fprintf(stderr, "Warning: path not on HugeTLBFS: %s\n", path);
2482 return fs.f_bsize;
2485 static void *file_ram_alloc(ram_addr_t memory, const char *path)
2487 char *filename;
2488 void *area;
2489 int fd;
2490 #ifdef MAP_POPULATE
2491 int flags;
2492 #endif
2493 unsigned long hpagesize;
2494 extern int mem_prealloc;
2496 if (!path) {
2497 return NULL;
2500 hpagesize = gethugepagesize(path);
2501 if (!hpagesize) {
2502 return NULL;
2505 if (memory < hpagesize) {
2506 return NULL;
2509 if (kvm_enabled() && !kvm_has_sync_mmu()) {
2510 fprintf(stderr, "host lacks mmu notifiers, disabling --mem-path\n");
2511 return NULL;
2514 if (asprintf(&filename, "%s/kvm.XXXXXX", path) == -1) {
2515 return NULL;
2518 fd = mkstemp(filename);
2519 if (fd < 0) {
2520 perror("mkstemp");
2521 free(filename);
2522 return NULL;
2524 unlink(filename);
2525 free(filename);
2527 memory = (memory+hpagesize-1) & ~(hpagesize-1);
2530 * ftruncate is not supported by hugetlbfs in older
2531 * hosts, so don't bother checking for errors.
2532 * If anything goes wrong with it under other filesystems,
2533 * mmap will fail.
2535 ftruncate(fd, memory);
2537 #ifdef MAP_POPULATE
2538 /* NB: MAP_POPULATE won't exhaustively alloc all phys pages in the case
2539 * MAP_PRIVATE is requested. For mem_prealloc we mmap as MAP_SHARED
2540 * to sidestep this quirk.
2542 flags = mem_prealloc ? MAP_POPULATE|MAP_SHARED : MAP_PRIVATE;
2543 area = mmap(0, memory, PROT_READ|PROT_WRITE, flags, fd, 0);
2544 #else
2545 area = mmap(0, memory, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);
2546 #endif
2547 if (area == MAP_FAILED) {
2548 perror("alloc_mem_area: can't mmap hugetlbfs pages");
2549 close(fd);
2550 return (NULL);
2552 return area;
2555 #else
2557 static void *file_ram_alloc(ram_addr_t memory, const char *path)
2559 return NULL;
2562 #endif
2564 extern const char *mem_path;
2566 ram_addr_t qemu_ram_alloc(ram_addr_t size)
2568 RAMBlock *new_block;
2570 #ifdef CONFIG_KQEMU
2571 if (kqemu_phys_ram_base) {
2572 return kqemu_ram_alloc(size);
2574 #endif
2576 size = TARGET_PAGE_ALIGN(size);
2577 new_block = qemu_malloc(sizeof(*new_block));
2579 new_block->host = file_ram_alloc(size, mem_path);
2580 if (!new_block->host) {
2581 new_block->host = qemu_vmalloc(size);
2583 new_block->offset = last_ram_offset;
2584 new_block->length = size;
2586 new_block->next = ram_blocks;
2587 ram_blocks = new_block;
2589 phys_ram_dirty = qemu_realloc(phys_ram_dirty,
2590 (last_ram_offset + size) >> TARGET_PAGE_BITS);
2591 memset(phys_ram_dirty + (last_ram_offset >> TARGET_PAGE_BITS),
2592 0xff, size >> TARGET_PAGE_BITS);
2594 last_ram_offset += size;
2596 if (kvm_enabled())
2597 kvm_setup_guest_memory(new_block->host, size);
2599 return new_block->offset;
2602 void qemu_ram_free(ram_addr_t addr)
2604 /* TODO: implement this. */
2607 /* Return a host pointer to ram allocated with qemu_ram_alloc.
2608 With the exception of the softmmu code in this file, this should
2609 only be used for local memory (e.g. video ram) that the device owns,
2610 and knows it isn't going to access beyond the end of the block.
2612 It should not be used for general purpose DMA.
2613 Use cpu_physical_memory_map/cpu_physical_memory_rw instead.
2615 void *qemu_get_ram_ptr(ram_addr_t addr)
2617 RAMBlock *prev;
2618 RAMBlock **prevp;
2619 RAMBlock *block;
2621 #ifdef CONFIG_KQEMU
2622 if (kqemu_phys_ram_base) {
2623 return kqemu_phys_ram_base + addr;
2625 #endif
2627 prev = NULL;
2628 prevp = &ram_blocks;
2629 block = ram_blocks;
2630 while (block && (block->offset > addr
2631 || block->offset + block->length <= addr)) {
2632 if (prev)
2633 prevp = &prev->next;
2634 prev = block;
2635 block = block->next;
2637 if (!block) {
2638 fprintf(stderr, "Bad ram offset %" PRIx64 "\n", (uint64_t)addr);
2639 abort();
2641 /* Move this entry to to start of the list. */
2642 if (prev) {
2643 prev->next = block->next;
2644 block->next = *prevp;
2645 *prevp = block;
2647 return block->host + (addr - block->offset);
2650 /* Some of the softmmu routines need to translate from a host pointer
2651 (typically a TLB entry) back to a ram offset. */
2652 ram_addr_t qemu_ram_addr_from_host(void *ptr)
2654 RAMBlock *prev;
2655 RAMBlock **prevp;
2656 RAMBlock *block;
2657 uint8_t *host = ptr;
2659 #ifdef CONFIG_KQEMU
2660 if (kqemu_phys_ram_base) {
2661 return host - kqemu_phys_ram_base;
2663 #endif
2665 prev = NULL;
2666 prevp = &ram_blocks;
2667 block = ram_blocks;
2668 while (block && (block->host > host
2669 || block->host + block->length <= host)) {
2670 if (prev)
2671 prevp = &prev->next;
2672 prev = block;
2673 block = block->next;
2675 if (!block) {
2676 fprintf(stderr, "Bad ram pointer %p\n", ptr);
2677 abort();
2679 return block->offset + (host - block->host);
2682 static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
2684 #ifdef DEBUG_UNASSIGNED
2685 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2686 #endif
2687 #if defined(TARGET_SPARC)
2688 do_unassigned_access(addr, 0, 0, 0, 1);
2689 #endif
2690 return 0;
2693 static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2695 #ifdef DEBUG_UNASSIGNED
2696 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2697 #endif
2698 #if defined(TARGET_SPARC)
2699 do_unassigned_access(addr, 0, 0, 0, 2);
2700 #endif
2701 return 0;
2704 static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2706 #ifdef DEBUG_UNASSIGNED
2707 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2708 #endif
2709 #if defined(TARGET_SPARC)
2710 do_unassigned_access(addr, 0, 0, 0, 4);
2711 #endif
2712 return 0;
2715 static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
2717 #ifdef DEBUG_UNASSIGNED
2718 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2719 #endif
2720 #if defined(TARGET_SPARC)
2721 do_unassigned_access(addr, 1, 0, 0, 1);
2722 #endif
2725 static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2727 #ifdef DEBUG_UNASSIGNED
2728 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2729 #endif
2730 #if defined(TARGET_SPARC)
2731 do_unassigned_access(addr, 1, 0, 0, 2);
2732 #endif
2735 static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2737 #ifdef DEBUG_UNASSIGNED
2738 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2739 #endif
2740 #if defined(TARGET_SPARC)
2741 do_unassigned_access(addr, 1, 0, 0, 4);
2742 #endif
2745 static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2746 unassigned_mem_readb,
2747 unassigned_mem_readw,
2748 unassigned_mem_readl,
2751 static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2752 unassigned_mem_writeb,
2753 unassigned_mem_writew,
2754 unassigned_mem_writel,
2757 static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2758 uint32_t val)
2760 int dirty_flags;
2761 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2762 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2763 #if !defined(CONFIG_USER_ONLY)
2764 tb_invalidate_phys_page_fast(ram_addr, 1);
2765 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2766 #endif
2768 stb_p(qemu_get_ram_ptr(ram_addr), val);
2769 #ifdef CONFIG_KQEMU
2770 if (cpu_single_env->kqemu_enabled &&
2771 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2772 kqemu_modify_page(cpu_single_env, ram_addr);
2773 #endif
2774 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2775 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2776 /* we remove the notdirty callback only if the code has been
2777 flushed */
2778 if (dirty_flags == 0xff)
2779 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2782 static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2783 uint32_t val)
2785 int dirty_flags;
2786 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2787 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2788 #if !defined(CONFIG_USER_ONLY)
2789 tb_invalidate_phys_page_fast(ram_addr, 2);
2790 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2791 #endif
2793 stw_p(qemu_get_ram_ptr(ram_addr), val);
2794 #ifdef CONFIG_KQEMU
2795 if (cpu_single_env->kqemu_enabled &&
2796 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2797 kqemu_modify_page(cpu_single_env, ram_addr);
2798 #endif
2799 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2800 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2801 /* we remove the notdirty callback only if the code has been
2802 flushed */
2803 if (dirty_flags == 0xff)
2804 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2807 static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2808 uint32_t val)
2810 int dirty_flags;
2811 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2812 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
2813 #if !defined(CONFIG_USER_ONLY)
2814 tb_invalidate_phys_page_fast(ram_addr, 4);
2815 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2816 #endif
2818 stl_p(qemu_get_ram_ptr(ram_addr), val);
2819 #ifdef CONFIG_KQEMU
2820 if (cpu_single_env->kqemu_enabled &&
2821 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2822 kqemu_modify_page(cpu_single_env, ram_addr);
2823 #endif
2824 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2825 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2826 /* we remove the notdirty callback only if the code has been
2827 flushed */
2828 if (dirty_flags == 0xff)
2829 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
2832 static CPUReadMemoryFunc *error_mem_read[3] = {
2833 NULL, /* never used */
2834 NULL, /* never used */
2835 NULL, /* never used */
2838 static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2839 notdirty_mem_writeb,
2840 notdirty_mem_writew,
2841 notdirty_mem_writel,
2844 /* Generate a debug exception if a watchpoint has been hit. */
2845 static void check_watchpoint(int offset, int len_mask, int flags)
2847 CPUState *env = cpu_single_env;
2848 target_ulong pc, cs_base;
2849 TranslationBlock *tb;
2850 target_ulong vaddr;
2851 CPUWatchpoint *wp;
2852 int cpu_flags;
2854 if (env->watchpoint_hit) {
2855 /* We re-entered the check after replacing the TB. Now raise
2856 * the debug interrupt so that is will trigger after the
2857 * current instruction. */
2858 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2859 return;
2861 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
2862 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
2863 if ((vaddr == (wp->vaddr & len_mask) ||
2864 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
2865 wp->flags |= BP_WATCHPOINT_HIT;
2866 if (!env->watchpoint_hit) {
2867 env->watchpoint_hit = wp;
2868 tb = tb_find_pc(env->mem_io_pc);
2869 if (!tb) {
2870 cpu_abort(env, "check_watchpoint: could not find TB for "
2871 "pc=%p", (void *)env->mem_io_pc);
2873 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2874 tb_phys_invalidate(tb, -1);
2875 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2876 env->exception_index = EXCP_DEBUG;
2877 } else {
2878 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2879 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2881 cpu_resume_from_signal(env, NULL);
2883 } else {
2884 wp->flags &= ~BP_WATCHPOINT_HIT;
2889 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2890 so these check for a hit then pass through to the normal out-of-line
2891 phys routines. */
2892 static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2894 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
2895 return ldub_phys(addr);
2898 static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2900 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
2901 return lduw_phys(addr);
2904 static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2906 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
2907 return ldl_phys(addr);
2910 static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2911 uint32_t val)
2913 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
2914 stb_phys(addr, val);
2917 static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2918 uint32_t val)
2920 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
2921 stw_phys(addr, val);
2924 static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2925 uint32_t val)
2927 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
2928 stl_phys(addr, val);
2931 static CPUReadMemoryFunc *watch_mem_read[3] = {
2932 watch_mem_readb,
2933 watch_mem_readw,
2934 watch_mem_readl,
2937 static CPUWriteMemoryFunc *watch_mem_write[3] = {
2938 watch_mem_writeb,
2939 watch_mem_writew,
2940 watch_mem_writel,
2943 static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2944 unsigned int len)
2946 uint32_t ret;
2947 unsigned int idx;
2949 idx = SUBPAGE_IDX(addr);
2950 #if defined(DEBUG_SUBPAGE)
2951 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2952 mmio, len, addr, idx);
2953 #endif
2954 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2955 addr + mmio->region_offset[idx][0][len]);
2957 return ret;
2960 static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2961 uint32_t value, unsigned int len)
2963 unsigned int idx;
2965 idx = SUBPAGE_IDX(addr);
2966 #if defined(DEBUG_SUBPAGE)
2967 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2968 mmio, len, addr, idx, value);
2969 #endif
2970 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2971 addr + mmio->region_offset[idx][1][len],
2972 value);
2975 static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2977 #if defined(DEBUG_SUBPAGE)
2978 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2979 #endif
2981 return subpage_readlen(opaque, addr, 0);
2984 static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2985 uint32_t value)
2987 #if defined(DEBUG_SUBPAGE)
2988 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2989 #endif
2990 subpage_writelen(opaque, addr, value, 0);
2993 static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2995 #if defined(DEBUG_SUBPAGE)
2996 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2997 #endif
2999 return subpage_readlen(opaque, addr, 1);
3002 static void subpage_writew (void *opaque, target_phys_addr_t addr,
3003 uint32_t value)
3005 #if defined(DEBUG_SUBPAGE)
3006 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3007 #endif
3008 subpage_writelen(opaque, addr, value, 1);
3011 static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
3013 #if defined(DEBUG_SUBPAGE)
3014 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
3015 #endif
3017 return subpage_readlen(opaque, addr, 2);
3020 static void subpage_writel (void *opaque,
3021 target_phys_addr_t addr, uint32_t value)
3023 #if defined(DEBUG_SUBPAGE)
3024 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
3025 #endif
3026 subpage_writelen(opaque, addr, value, 2);
3029 static CPUReadMemoryFunc *subpage_read[] = {
3030 &subpage_readb,
3031 &subpage_readw,
3032 &subpage_readl,
3035 static CPUWriteMemoryFunc *subpage_write[] = {
3036 &subpage_writeb,
3037 &subpage_writew,
3038 &subpage_writel,
3041 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
3042 ram_addr_t memory, ram_addr_t region_offset)
3044 int idx, eidx;
3045 unsigned int i;
3047 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
3048 return -1;
3049 idx = SUBPAGE_IDX(start);
3050 eidx = SUBPAGE_IDX(end);
3051 #if defined(DEBUG_SUBPAGE)
3052 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
3053 mmio, start, end, idx, eidx, memory);
3054 #endif
3055 memory >>= IO_MEM_SHIFT;
3056 for (; idx <= eidx; idx++) {
3057 for (i = 0; i < 4; i++) {
3058 if (io_mem_read[memory][i]) {
3059 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
3060 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
3061 mmio->region_offset[idx][0][i] = region_offset;
3063 if (io_mem_write[memory][i]) {
3064 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
3065 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
3066 mmio->region_offset[idx][1][i] = region_offset;
3071 return 0;
3074 static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
3075 ram_addr_t orig_memory, ram_addr_t region_offset)
3077 subpage_t *mmio;
3078 int subpage_memory;
3080 mmio = qemu_mallocz(sizeof(subpage_t));
3082 mmio->base = base;
3083 subpage_memory = cpu_register_io_memory(subpage_read, subpage_write, mmio);
3084 #if defined(DEBUG_SUBPAGE)
3085 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
3086 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
3087 #endif
3088 *phys = subpage_memory | IO_MEM_SUBPAGE;
3089 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
3090 region_offset);
3092 return mmio;
3095 static int get_free_io_mem_idx(void)
3097 int i;
3099 for (i = 0; i<IO_MEM_NB_ENTRIES; i++)
3100 if (!io_mem_used[i]) {
3101 io_mem_used[i] = 1;
3102 return i;
3105 return -1;
3108 /* mem_read and mem_write are arrays of functions containing the
3109 function to access byte (index 0), word (index 1) and dword (index
3110 2). Functions can be omitted with a NULL function pointer.
3111 If io_index is non zero, the corresponding io zone is
3112 modified. If it is zero, a new io zone is allocated. The return
3113 value can be used with cpu_register_physical_memory(). (-1) is
3114 returned if error. */
3115 static int cpu_register_io_memory_fixed(int io_index,
3116 CPUReadMemoryFunc **mem_read,
3117 CPUWriteMemoryFunc **mem_write,
3118 void *opaque)
3120 int i, subwidth = 0;
3122 if (io_index <= 0) {
3123 io_index = get_free_io_mem_idx();
3124 if (io_index == -1)
3125 return io_index;
3126 } else {
3127 io_index >>= IO_MEM_SHIFT;
3128 if (io_index >= IO_MEM_NB_ENTRIES)
3129 return -1;
3132 for(i = 0;i < 3; i++) {
3133 if (!mem_read[i] || !mem_write[i])
3134 subwidth = IO_MEM_SUBWIDTH;
3135 io_mem_read[io_index][i] = mem_read[i];
3136 io_mem_write[io_index][i] = mem_write[i];
3138 io_mem_opaque[io_index] = opaque;
3139 return (io_index << IO_MEM_SHIFT) | subwidth;
3142 int cpu_register_io_memory(CPUReadMemoryFunc **mem_read,
3143 CPUWriteMemoryFunc **mem_write,
3144 void *opaque)
3146 return cpu_register_io_memory_fixed(0, mem_read, mem_write, opaque);
3149 void cpu_unregister_io_memory(int io_table_address)
3151 int i;
3152 int io_index = io_table_address >> IO_MEM_SHIFT;
3154 for (i=0;i < 3; i++) {
3155 io_mem_read[io_index][i] = unassigned_mem_read[i];
3156 io_mem_write[io_index][i] = unassigned_mem_write[i];
3158 io_mem_opaque[io_index] = NULL;
3159 io_mem_used[io_index] = 0;
3162 static void io_mem_init(void)
3164 int i;
3166 cpu_register_io_memory_fixed(IO_MEM_ROM, error_mem_read, unassigned_mem_write, NULL);
3167 cpu_register_io_memory_fixed(IO_MEM_UNASSIGNED, unassigned_mem_read, unassigned_mem_write, NULL);
3168 cpu_register_io_memory_fixed(IO_MEM_NOTDIRTY, error_mem_read, notdirty_mem_write, NULL);
3169 for (i=0; i<5; i++)
3170 io_mem_used[i] = 1;
3172 io_mem_watch = cpu_register_io_memory(watch_mem_read,
3173 watch_mem_write, NULL);
3174 #ifdef CONFIG_KQEMU
3175 if (kqemu_phys_ram_base) {
3176 /* alloc dirty bits array */
3177 phys_ram_dirty = qemu_vmalloc(kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3178 memset(phys_ram_dirty, 0xff, kqemu_phys_ram_size >> TARGET_PAGE_BITS);
3180 #endif
3183 #endif /* !defined(CONFIG_USER_ONLY) */
3185 /* physical memory access (slow version, mainly for debug) */
3186 #if defined(CONFIG_USER_ONLY)
3187 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3188 int len, int is_write)
3190 int l, flags;
3191 target_ulong page;
3192 void * p;
3194 while (len > 0) {
3195 page = addr & TARGET_PAGE_MASK;
3196 l = (page + TARGET_PAGE_SIZE) - addr;
3197 if (l > len)
3198 l = len;
3199 flags = page_get_flags(page);
3200 if (!(flags & PAGE_VALID))
3201 return;
3202 if (is_write) {
3203 if (!(flags & PAGE_WRITE))
3204 return;
3205 /* XXX: this code should not depend on lock_user */
3206 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
3207 /* FIXME - should this return an error rather than just fail? */
3208 return;
3209 memcpy(p, buf, l);
3210 unlock_user(p, addr, l);
3211 } else {
3212 if (!(flags & PAGE_READ))
3213 return;
3214 /* XXX: this code should not depend on lock_user */
3215 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
3216 /* FIXME - should this return an error rather than just fail? */
3217 return;
3218 memcpy(buf, p, l);
3219 unlock_user(p, addr, 0);
3221 len -= l;
3222 buf += l;
3223 addr += l;
3227 #else
3228 void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
3229 int len, int is_write)
3231 int l, io_index;
3232 uint8_t *ptr;
3233 uint32_t val;
3234 target_phys_addr_t page;
3235 unsigned long pd;
3236 PhysPageDesc *p;
3238 while (len > 0) {
3239 page = addr & TARGET_PAGE_MASK;
3240 l = (page + TARGET_PAGE_SIZE) - addr;
3241 if (l > len)
3242 l = len;
3243 p = phys_page_find(page >> TARGET_PAGE_BITS);
3244 if (!p) {
3245 pd = IO_MEM_UNASSIGNED;
3246 } else {
3247 pd = p->phys_offset;
3250 if (is_write) {
3251 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3252 target_phys_addr_t addr1 = addr;
3253 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3254 if (p)
3255 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3256 /* XXX: could force cpu_single_env to NULL to avoid
3257 potential bugs */
3258 if (l >= 4 && ((addr1 & 3) == 0)) {
3259 /* 32 bit write access */
3260 val = ldl_p(buf);
3261 io_mem_write[io_index][2](io_mem_opaque[io_index], addr1, val);
3262 l = 4;
3263 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3264 /* 16 bit write access */
3265 val = lduw_p(buf);
3266 io_mem_write[io_index][1](io_mem_opaque[io_index], addr1, val);
3267 l = 2;
3268 } else {
3269 /* 8 bit write access */
3270 val = ldub_p(buf);
3271 io_mem_write[io_index][0](io_mem_opaque[io_index], addr1, val);
3272 l = 1;
3274 } else {
3275 unsigned long addr1;
3276 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3277 /* RAM case */
3278 ptr = qemu_get_ram_ptr(addr1);
3279 memcpy(ptr, buf, l);
3280 if (!cpu_physical_memory_is_dirty(addr1)) {
3281 /* invalidate code */
3282 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3283 /* set dirty bit */
3284 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3285 (0xff & ~CODE_DIRTY_FLAG);
3287 /* qemu doesn't execute guest code directly, but kvm does
3288 therefore flush instruction caches */
3289 if (kvm_enabled())
3290 flush_icache_range((unsigned long)ptr,
3291 ((unsigned long)ptr)+l);
3293 } else {
3294 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3295 !(pd & IO_MEM_ROMD)) {
3296 target_phys_addr_t addr1 = addr;
3297 /* I/O case */
3298 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3299 if (p)
3300 addr1 = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3301 if (l >= 4 && ((addr1 & 3) == 0)) {
3302 /* 32 bit read access */
3303 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr1);
3304 stl_p(buf, val);
3305 l = 4;
3306 } else if (l >= 2 && ((addr1 & 1) == 0)) {
3307 /* 16 bit read access */
3308 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr1);
3309 stw_p(buf, val);
3310 l = 2;
3311 } else {
3312 /* 8 bit read access */
3313 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr1);
3314 stb_p(buf, val);
3315 l = 1;
3317 } else {
3318 /* RAM case */
3319 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3320 (addr & ~TARGET_PAGE_MASK);
3321 memcpy(buf, ptr, l);
3324 len -= l;
3325 buf += l;
3326 addr += l;
3330 /* used for ROM loading : can write in RAM and ROM */
3331 void cpu_physical_memory_write_rom(target_phys_addr_t addr,
3332 const uint8_t *buf, int len)
3334 int l;
3335 uint8_t *ptr;
3336 target_phys_addr_t page;
3337 unsigned long pd;
3338 PhysPageDesc *p;
3340 while (len > 0) {
3341 page = addr & TARGET_PAGE_MASK;
3342 l = (page + TARGET_PAGE_SIZE) - addr;
3343 if (l > len)
3344 l = len;
3345 p = phys_page_find(page >> TARGET_PAGE_BITS);
3346 if (!p) {
3347 pd = IO_MEM_UNASSIGNED;
3348 } else {
3349 pd = p->phys_offset;
3352 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
3353 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3354 !(pd & IO_MEM_ROMD)) {
3355 /* do nothing */
3356 } else {
3357 unsigned long addr1;
3358 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3359 /* ROM/RAM case */
3360 ptr = qemu_get_ram_ptr(addr1);
3361 memcpy(ptr, buf, l);
3363 len -= l;
3364 buf += l;
3365 addr += l;
3369 typedef struct {
3370 void *buffer;
3371 target_phys_addr_t addr;
3372 target_phys_addr_t len;
3373 } BounceBuffer;
3375 static BounceBuffer bounce;
3377 typedef struct MapClient {
3378 void *opaque;
3379 void (*callback)(void *opaque);
3380 LIST_ENTRY(MapClient) link;
3381 } MapClient;
3383 static LIST_HEAD(map_client_list, MapClient) map_client_list
3384 = LIST_HEAD_INITIALIZER(map_client_list);
3386 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3388 MapClient *client = qemu_malloc(sizeof(*client));
3390 client->opaque = opaque;
3391 client->callback = callback;
3392 LIST_INSERT_HEAD(&map_client_list, client, link);
3393 return client;
3396 void cpu_unregister_map_client(void *_client)
3398 MapClient *client = (MapClient *)_client;
3400 LIST_REMOVE(client, link);
3401 qemu_free(client);
3404 static void cpu_notify_map_clients(void)
3406 MapClient *client;
3408 while (!LIST_EMPTY(&map_client_list)) {
3409 client = LIST_FIRST(&map_client_list);
3410 client->callback(client->opaque);
3411 cpu_unregister_map_client(client);
3415 /* Map a physical memory region into a host virtual address.
3416 * May map a subset of the requested range, given by and returned in *plen.
3417 * May return NULL if resources needed to perform the mapping are exhausted.
3418 * Use only for reads OR writes - not for read-modify-write operations.
3419 * Use cpu_register_map_client() to know when retrying the map operation is
3420 * likely to succeed.
3422 void *cpu_physical_memory_map(target_phys_addr_t addr,
3423 target_phys_addr_t *plen,
3424 int is_write)
3426 target_phys_addr_t len = *plen;
3427 target_phys_addr_t done = 0;
3428 int l;
3429 uint8_t *ret = NULL;
3430 uint8_t *ptr;
3431 target_phys_addr_t page;
3432 unsigned long pd;
3433 PhysPageDesc *p;
3434 unsigned long addr1;
3436 while (len > 0) {
3437 page = addr & TARGET_PAGE_MASK;
3438 l = (page + TARGET_PAGE_SIZE) - addr;
3439 if (l > len)
3440 l = len;
3441 p = phys_page_find(page >> TARGET_PAGE_BITS);
3442 if (!p) {
3443 pd = IO_MEM_UNASSIGNED;
3444 } else {
3445 pd = p->phys_offset;
3448 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3449 if (done || bounce.buffer) {
3450 break;
3452 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3453 bounce.addr = addr;
3454 bounce.len = l;
3455 if (!is_write) {
3456 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3458 ptr = bounce.buffer;
3459 } else {
3460 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3461 ptr = qemu_get_ram_ptr(addr1);
3463 if (!done) {
3464 ret = ptr;
3465 } else if (ret + done != ptr) {
3466 break;
3469 len -= l;
3470 addr += l;
3471 done += l;
3473 *plen = done;
3474 return ret;
3477 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3478 * Will also mark the memory as dirty if is_write == 1. access_len gives
3479 * the amount of memory that was actually read or written by the caller.
3481 void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3482 int is_write, target_phys_addr_t access_len)
3484 unsigned long flush_len = (unsigned long)access_len;
3486 if (buffer != bounce.buffer) {
3487 if (is_write) {
3488 ram_addr_t addr1 = qemu_ram_addr_from_host(buffer);
3489 while (access_len) {
3490 unsigned l;
3491 l = TARGET_PAGE_SIZE;
3492 if (l > access_len)
3493 l = access_len;
3494 if (!cpu_physical_memory_is_dirty(addr1)) {
3495 /* invalidate code */
3496 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3497 /* set dirty bit */
3498 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3499 (0xff & ~CODE_DIRTY_FLAG);
3501 addr1 += l;
3502 access_len -= l;
3504 dma_flush_range((unsigned long)buffer,
3505 (unsigned long)buffer + flush_len);
3507 return;
3509 if (is_write) {
3510 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3512 qemu_free(bounce.buffer);
3513 bounce.buffer = NULL;
3514 cpu_notify_map_clients();
3517 /* warning: addr must be aligned */
3518 uint32_t ldl_phys(target_phys_addr_t addr)
3520 int io_index;
3521 uint8_t *ptr;
3522 uint32_t val;
3523 unsigned long pd;
3524 PhysPageDesc *p;
3526 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3527 if (!p) {
3528 pd = IO_MEM_UNASSIGNED;
3529 } else {
3530 pd = p->phys_offset;
3533 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3534 !(pd & IO_MEM_ROMD)) {
3535 /* I/O case */
3536 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3537 if (p)
3538 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3539 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3540 } else {
3541 /* RAM case */
3542 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3543 (addr & ~TARGET_PAGE_MASK);
3544 val = ldl_p(ptr);
3546 return val;
3549 /* warning: addr must be aligned */
3550 uint64_t ldq_phys(target_phys_addr_t addr)
3552 int io_index;
3553 uint8_t *ptr;
3554 uint64_t val;
3555 unsigned long pd;
3556 PhysPageDesc *p;
3558 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3559 if (!p) {
3560 pd = IO_MEM_UNASSIGNED;
3561 } else {
3562 pd = p->phys_offset;
3565 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3566 !(pd & IO_MEM_ROMD)) {
3567 /* I/O case */
3568 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3569 if (p)
3570 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3571 #ifdef TARGET_WORDS_BIGENDIAN
3572 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3573 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3574 #else
3575 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3576 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3577 #endif
3578 } else {
3579 /* RAM case */
3580 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3581 (addr & ~TARGET_PAGE_MASK);
3582 val = ldq_p(ptr);
3584 return val;
3587 /* XXX: optimize */
3588 uint32_t ldub_phys(target_phys_addr_t addr)
3590 uint8_t val;
3591 cpu_physical_memory_read(addr, &val, 1);
3592 return val;
3595 /* XXX: optimize */
3596 uint32_t lduw_phys(target_phys_addr_t addr)
3598 uint16_t val;
3599 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3600 return tswap16(val);
3603 /* warning: addr must be aligned. The ram page is not masked as dirty
3604 and the code inside is not invalidated. It is useful if the dirty
3605 bits are used to track modified PTEs */
3606 void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3608 int io_index;
3609 uint8_t *ptr;
3610 unsigned long pd;
3611 PhysPageDesc *p;
3613 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3614 if (!p) {
3615 pd = IO_MEM_UNASSIGNED;
3616 } else {
3617 pd = p->phys_offset;
3620 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3621 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3622 if (p)
3623 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3624 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3625 } else {
3626 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3627 ptr = qemu_get_ram_ptr(addr1);
3628 stl_p(ptr, val);
3630 if (unlikely(in_migration)) {
3631 if (!cpu_physical_memory_is_dirty(addr1)) {
3632 /* invalidate code */
3633 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3634 /* set dirty bit */
3635 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3636 (0xff & ~CODE_DIRTY_FLAG);
3642 void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3644 int io_index;
3645 uint8_t *ptr;
3646 unsigned long pd;
3647 PhysPageDesc *p;
3649 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3650 if (!p) {
3651 pd = IO_MEM_UNASSIGNED;
3652 } else {
3653 pd = p->phys_offset;
3656 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3657 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3658 if (p)
3659 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3660 #ifdef TARGET_WORDS_BIGENDIAN
3661 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3662 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3663 #else
3664 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3665 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3666 #endif
3667 } else {
3668 ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
3669 (addr & ~TARGET_PAGE_MASK);
3670 stq_p(ptr, val);
3674 /* warning: addr must be aligned */
3675 void stl_phys(target_phys_addr_t addr, uint32_t val)
3677 int io_index;
3678 uint8_t *ptr;
3679 unsigned long pd;
3680 PhysPageDesc *p;
3682 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3683 if (!p) {
3684 pd = IO_MEM_UNASSIGNED;
3685 } else {
3686 pd = p->phys_offset;
3689 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3690 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
3691 if (p)
3692 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
3693 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3694 } else {
3695 unsigned long addr1;
3696 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3697 /* RAM case */
3698 ptr = qemu_get_ram_ptr(addr1);
3699 stl_p(ptr, val);
3700 if (!cpu_physical_memory_is_dirty(addr1)) {
3701 /* invalidate code */
3702 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3703 /* set dirty bit */
3704 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3705 (0xff & ~CODE_DIRTY_FLAG);
3710 /* XXX: optimize */
3711 void stb_phys(target_phys_addr_t addr, uint32_t val)
3713 uint8_t v = val;
3714 cpu_physical_memory_write(addr, &v, 1);
3717 /* XXX: optimize */
3718 void stw_phys(target_phys_addr_t addr, uint32_t val)
3720 uint16_t v = tswap16(val);
3721 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3724 /* XXX: optimize */
3725 void stq_phys(target_phys_addr_t addr, uint64_t val)
3727 val = tswap64(val);
3728 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3731 #endif
3733 /* virtual memory access for debug (includes writing to ROM) */
3734 int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
3735 uint8_t *buf, int len, int is_write)
3737 int l;
3738 target_phys_addr_t phys_addr;
3739 target_ulong page;
3741 while (len > 0) {
3742 page = addr & TARGET_PAGE_MASK;
3743 phys_addr = cpu_get_phys_page_debug(env, page);
3744 /* if no physical page mapped, return an error */
3745 if (phys_addr == -1)
3746 return -1;
3747 l = (page + TARGET_PAGE_SIZE) - addr;
3748 if (l > len)
3749 l = len;
3750 phys_addr += (addr & ~TARGET_PAGE_MASK);
3751 #if !defined(CONFIG_USER_ONLY)
3752 if (is_write)
3753 cpu_physical_memory_write_rom(phys_addr, buf, l);
3754 else
3755 #endif
3756 cpu_physical_memory_rw(phys_addr, buf, l, is_write);
3757 len -= l;
3758 buf += l;
3759 addr += l;
3761 return 0;
3764 /* in deterministic execution mode, instructions doing device I/Os
3765 must be at the end of the TB */
3766 void cpu_io_recompile(CPUState *env, void *retaddr)
3768 TranslationBlock *tb;
3769 uint32_t n, cflags;
3770 target_ulong pc, cs_base;
3771 uint64_t flags;
3773 tb = tb_find_pc((unsigned long)retaddr);
3774 if (!tb) {
3775 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3776 retaddr);
3778 n = env->icount_decr.u16.low + tb->icount;
3779 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3780 /* Calculate how many instructions had been executed before the fault
3781 occurred. */
3782 n = n - env->icount_decr.u16.low;
3783 /* Generate a new TB ending on the I/O insn. */
3784 n++;
3785 /* On MIPS and SH, delay slot instructions can only be restarted if
3786 they were already the first instruction in the TB. If this is not
3787 the first instruction in a TB then re-execute the preceding
3788 branch. */
3789 #if defined(TARGET_MIPS)
3790 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3791 env->active_tc.PC -= 4;
3792 env->icount_decr.u16.low++;
3793 env->hflags &= ~MIPS_HFLAG_BMASK;
3795 #elif defined(TARGET_SH4)
3796 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3797 && n > 1) {
3798 env->pc -= 2;
3799 env->icount_decr.u16.low++;
3800 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3802 #endif
3803 /* This should never happen. */
3804 if (n > CF_COUNT_MASK)
3805 cpu_abort(env, "TB too big during recompile");
3807 cflags = n | CF_LAST_IO;
3808 pc = tb->pc;
3809 cs_base = tb->cs_base;
3810 flags = tb->flags;
3811 tb_phys_invalidate(tb, -1);
3812 /* FIXME: In theory this could raise an exception. In practice
3813 we have already translated the block once so it's probably ok. */
3814 tb_gen_code(env, pc, cs_base, flags, cflags);
3815 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3816 the first in the TB) then we end up generating a whole new TB and
3817 repeating the fault, which is horribly inefficient.
3818 Better would be to execute just this insn uncached, or generate a
3819 second new TB. */
3820 cpu_resume_from_signal(env, NULL);
3823 void dump_exec_info(FILE *f,
3824 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3826 int i, target_code_size, max_target_code_size;
3827 int direct_jmp_count, direct_jmp2_count, cross_page;
3828 TranslationBlock *tb;
3830 target_code_size = 0;
3831 max_target_code_size = 0;
3832 cross_page = 0;
3833 direct_jmp_count = 0;
3834 direct_jmp2_count = 0;
3835 for(i = 0; i < nb_tbs; i++) {
3836 tb = &tbs[i];
3837 target_code_size += tb->size;
3838 if (tb->size > max_target_code_size)
3839 max_target_code_size = tb->size;
3840 if (tb->page_addr[1] != -1)
3841 cross_page++;
3842 if (tb->tb_next_offset[0] != 0xffff) {
3843 direct_jmp_count++;
3844 if (tb->tb_next_offset[1] != 0xffff) {
3845 direct_jmp2_count++;
3849 /* XXX: avoid using doubles ? */
3850 cpu_fprintf(f, "Translation buffer state:\n");
3851 cpu_fprintf(f, "gen code size %ld/%ld\n",
3852 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3853 cpu_fprintf(f, "TB count %d/%d\n",
3854 nb_tbs, code_gen_max_blocks);
3855 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
3856 nb_tbs ? target_code_size / nb_tbs : 0,
3857 max_target_code_size);
3858 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3859 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3860 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
3861 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3862 cross_page,
3863 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3864 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3865 direct_jmp_count,
3866 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3867 direct_jmp2_count,
3868 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
3869 cpu_fprintf(f, "\nStatistics:\n");
3870 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3871 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3872 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
3873 tcg_dump_info(f, cpu_fprintf);
3876 #if !defined(CONFIG_USER_ONLY)
3878 #define MMUSUFFIX _cmmu
3879 #define GETPC() NULL
3880 #define env cpu_single_env
3881 #define SOFTMMU_CODE_ACCESS
3883 #define SHIFT 0
3884 #include "softmmu_template.h"
3886 #define SHIFT 1
3887 #include "softmmu_template.h"
3889 #define SHIFT 2
3890 #include "softmmu_template.h"
3892 #define SHIFT 3
3893 #include "softmmu_template.h"
3895 #undef env
3897 #endif