2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
22 #define WIN32_LEAN_AND_MEAN
25 #include <sys/types.h>
38 #include "qemu-common.h"
43 #if defined(CONFIG_USER_ONLY)
47 //#define DEBUG_TB_INVALIDATE
50 //#define DEBUG_UNASSIGNED
52 /* make various TB consistency checks */
53 //#define DEBUG_TB_CHECK
54 //#define DEBUG_TLB_CHECK
56 //#define DEBUG_IOPORT
57 //#define DEBUG_SUBPAGE
59 #if !defined(CONFIG_USER_ONLY)
60 /* TB consistency checks only implemented for usermode emulation. */
64 #define SMC_BITMAP_USE_THRESHOLD 10
66 #if defined(TARGET_SPARC64)
67 #define TARGET_PHYS_ADDR_SPACE_BITS 41
68 #elif defined(TARGET_SPARC)
69 #define TARGET_PHYS_ADDR_SPACE_BITS 36
70 #elif defined(TARGET_ALPHA)
71 #define TARGET_PHYS_ADDR_SPACE_BITS 42
72 #define TARGET_VIRT_ADDR_SPACE_BITS 42
73 #elif defined(TARGET_PPC64)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 42
75 #elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
76 #define TARGET_PHYS_ADDR_SPACE_BITS 42
77 #elif defined(TARGET_I386) && !defined(USE_KQEMU)
78 #define TARGET_PHYS_ADDR_SPACE_BITS 36
80 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
81 #define TARGET_PHYS_ADDR_SPACE_BITS 32
84 static TranslationBlock
*tbs
;
85 int code_gen_max_blocks
;
86 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
88 /* any access to the tbs or the page table must use this lock */
89 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
91 #if defined(__arm__) || defined(__sparc_v9__)
92 /* The prologue must be reachable with a direct jump. ARM and Sparc64
93 have limited branch ranges (possibly also PPC) so place it in a
94 section close to code segment. */
95 #define code_gen_section \
96 __attribute__((__section__(".gen_code"))) \
97 __attribute__((aligned (32)))
99 #define code_gen_section \
100 __attribute__((aligned (32)))
103 uint8_t code_gen_prologue
[1024] code_gen_section
;
104 static uint8_t *code_gen_buffer
;
105 static unsigned long code_gen_buffer_size
;
106 /* threshold to flush the translated code buffer */
107 static unsigned long code_gen_buffer_max_size
;
108 uint8_t *code_gen_ptr
;
110 #if !defined(CONFIG_USER_ONLY)
111 ram_addr_t phys_ram_size
;
113 uint8_t *phys_ram_base
;
114 uint8_t *phys_ram_dirty
;
115 static int in_migration
;
116 static ram_addr_t phys_ram_alloc_offset
= 0;
120 /* current CPU in the current thread. It is only valid inside
122 CPUState
*cpu_single_env
;
123 /* 0 = Do not count executed instructions.
124 1 = Precise instruction counting.
125 2 = Adaptive rate instruction counting. */
127 /* Current instruction counter. While executing translated code this may
128 include some instructions that have not yet been executed. */
131 typedef struct PageDesc
{
132 /* list of TBs intersecting this ram page */
133 TranslationBlock
*first_tb
;
134 /* in order to optimize self modifying code, we count the number
135 of lookups we do to a given page to use a bitmap */
136 unsigned int code_write_count
;
137 uint8_t *code_bitmap
;
138 #if defined(CONFIG_USER_ONLY)
143 typedef struct PhysPageDesc
{
144 /* offset in host memory of the page + io_index in the low bits */
145 ram_addr_t phys_offset
;
146 ram_addr_t region_offset
;
150 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
151 /* XXX: this is a temporary hack for alpha target.
152 * In the future, this is to be replaced by a multi-level table
153 * to actually be able to handle the complete 64 bits address space.
155 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
157 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
160 #define L1_SIZE (1 << L1_BITS)
161 #define L2_SIZE (1 << L2_BITS)
163 unsigned long qemu_real_host_page_size
;
164 unsigned long qemu_host_page_bits
;
165 unsigned long qemu_host_page_size
;
166 unsigned long qemu_host_page_mask
;
168 /* XXX: for system emulation, it could just be an array */
169 static PageDesc
*l1_map
[L1_SIZE
];
170 static PhysPageDesc
**l1_phys_map
;
172 #if !defined(CONFIG_USER_ONLY)
173 static void io_mem_init(void);
175 /* io memory support */
176 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
177 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
178 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
179 static char io_mem_used
[IO_MEM_NB_ENTRIES
];
180 static int io_mem_watch
;
184 static const char *logfilename
= "/tmp/qemu.log";
187 static int log_append
= 0;
190 static int tlb_flush_count
;
191 static int tb_flush_count
;
192 static int tb_phys_invalidate_count
;
194 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
195 typedef struct subpage_t
{
196 target_phys_addr_t base
;
197 CPUReadMemoryFunc
**mem_read
[TARGET_PAGE_SIZE
][4];
198 CPUWriteMemoryFunc
**mem_write
[TARGET_PAGE_SIZE
][4];
199 void *opaque
[TARGET_PAGE_SIZE
][2][4];
200 ram_addr_t region_offset
[TARGET_PAGE_SIZE
][2][4];
204 static void map_exec(void *addr
, long size
)
207 VirtualProtect(addr
, size
,
208 PAGE_EXECUTE_READWRITE
, &old_protect
);
212 static void map_exec(void *addr
, long size
)
214 unsigned long start
, end
, page_size
;
216 page_size
= getpagesize();
217 start
= (unsigned long)addr
;
218 start
&= ~(page_size
- 1);
220 end
= (unsigned long)addr
+ size
;
221 end
+= page_size
- 1;
222 end
&= ~(page_size
- 1);
224 mprotect((void *)start
, end
- start
,
225 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
229 static void page_init(void)
231 /* NOTE: we can always suppose that qemu_host_page_size >=
235 SYSTEM_INFO system_info
;
237 GetSystemInfo(&system_info
);
238 qemu_real_host_page_size
= system_info
.dwPageSize
;
241 qemu_real_host_page_size
= getpagesize();
243 if (qemu_host_page_size
== 0)
244 qemu_host_page_size
= qemu_real_host_page_size
;
245 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
246 qemu_host_page_size
= TARGET_PAGE_SIZE
;
247 qemu_host_page_bits
= 0;
248 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
249 qemu_host_page_bits
++;
250 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
251 l1_phys_map
= qemu_vmalloc(L1_SIZE
* sizeof(void *));
252 memset(l1_phys_map
, 0, L1_SIZE
* sizeof(void *));
254 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
256 long long startaddr
, endaddr
;
261 last_brk
= (unsigned long)sbrk(0);
262 f
= fopen("/proc/self/maps", "r");
265 n
= fscanf (f
, "%llx-%llx %*[^\n]\n", &startaddr
, &endaddr
);
267 startaddr
= MIN(startaddr
,
268 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
269 endaddr
= MIN(endaddr
,
270 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS
) - 1);
271 page_set_flags(startaddr
& TARGET_PAGE_MASK
,
272 TARGET_PAGE_ALIGN(endaddr
),
283 static inline PageDesc
**page_l1_map(target_ulong index
)
285 #if TARGET_LONG_BITS > 32
286 /* Host memory outside guest VM. For 32-bit targets we have already
287 excluded high addresses. */
288 if (index
> ((target_ulong
)L2_SIZE
* L1_SIZE
))
291 return &l1_map
[index
>> L2_BITS
];
294 static inline PageDesc
*page_find_alloc(target_ulong index
)
297 lp
= page_l1_map(index
);
303 /* allocate if not found */
304 #if defined(CONFIG_USER_ONLY)
305 size_t len
= sizeof(PageDesc
) * L2_SIZE
;
306 /* Don't use qemu_malloc because it may recurse. */
307 p
= mmap(0, len
, PROT_READ
| PROT_WRITE
,
308 MAP_PRIVATE
| MAP_ANONYMOUS
, -1, 0);
311 unsigned long addr
= h2g(p
);
312 page_set_flags(addr
& TARGET_PAGE_MASK
,
313 TARGET_PAGE_ALIGN(addr
+ len
),
317 p
= qemu_mallocz(sizeof(PageDesc
) * L2_SIZE
);
321 return p
+ (index
& (L2_SIZE
- 1));
324 static inline PageDesc
*page_find(target_ulong index
)
327 lp
= page_l1_map(index
);
334 return p
+ (index
& (L2_SIZE
- 1));
337 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
342 p
= (void **)l1_phys_map
;
343 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
345 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
346 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
348 lp
= p
+ ((index
>> (L1_BITS
+ L2_BITS
)) & (L1_SIZE
- 1));
351 /* allocate if not found */
354 p
= qemu_vmalloc(sizeof(void *) * L1_SIZE
);
355 memset(p
, 0, sizeof(void *) * L1_SIZE
);
359 lp
= p
+ ((index
>> L2_BITS
) & (L1_SIZE
- 1));
363 /* allocate if not found */
366 pd
= qemu_vmalloc(sizeof(PhysPageDesc
) * L2_SIZE
);
368 for (i
= 0; i
< L2_SIZE
; i
++) {
369 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
370 pd
[i
].region_offset
= (index
+ i
) << TARGET_PAGE_BITS
;
373 return ((PhysPageDesc
*)pd
) + (index
& (L2_SIZE
- 1));
376 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
378 return phys_page_find_alloc(index
, 0);
381 #if !defined(CONFIG_USER_ONLY)
382 static void tlb_protect_code(ram_addr_t ram_addr
);
383 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
385 #define mmap_lock() do { } while(0)
386 #define mmap_unlock() do { } while(0)
389 #define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
391 #if defined(CONFIG_USER_ONLY)
392 /* Currently it is not recommanded to allocate big chunks of data in
393 user mode. It will change when a dedicated libc will be used */
394 #define USE_STATIC_CODE_GEN_BUFFER
397 #ifdef USE_STATIC_CODE_GEN_BUFFER
398 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
];
401 static void code_gen_alloc(unsigned long tb_size
)
403 #ifdef USE_STATIC_CODE_GEN_BUFFER
404 code_gen_buffer
= static_code_gen_buffer
;
405 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
406 map_exec(code_gen_buffer
, code_gen_buffer_size
);
408 code_gen_buffer_size
= tb_size
;
409 if (code_gen_buffer_size
== 0) {
410 #if defined(CONFIG_USER_ONLY)
411 /* in user mode, phys_ram_size is not meaningful */
412 code_gen_buffer_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
414 /* XXX: needs ajustments */
415 code_gen_buffer_size
= (unsigned long)(phys_ram_size
/ 4);
418 if (code_gen_buffer_size
< MIN_CODE_GEN_BUFFER_SIZE
)
419 code_gen_buffer_size
= MIN_CODE_GEN_BUFFER_SIZE
;
420 /* The code gen buffer location may have constraints depending on
421 the host cpu and OS */
422 #if defined(__linux__)
427 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
428 #if defined(__x86_64__)
430 /* Cannot map more than that */
431 if (code_gen_buffer_size
> (800 * 1024 * 1024))
432 code_gen_buffer_size
= (800 * 1024 * 1024);
433 #elif defined(__sparc_v9__)
434 // Map the buffer below 2G, so we can use direct calls and branches
436 start
= (void *) 0x60000000UL
;
437 if (code_gen_buffer_size
> (512 * 1024 * 1024))
438 code_gen_buffer_size
= (512 * 1024 * 1024);
439 #elif defined(__arm__)
440 /* Map the buffer below 32M, so we can use direct calls and branches */
442 start
= (void *) 0x01000000UL
;
443 if (code_gen_buffer_size
> 16 * 1024 * 1024)
444 code_gen_buffer_size
= 16 * 1024 * 1024;
446 code_gen_buffer
= mmap(start
, code_gen_buffer_size
,
447 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
449 if (code_gen_buffer
== MAP_FAILED
) {
450 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
454 #elif defined(__FreeBSD__) || defined(__DragonFly__)
458 flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
459 #if defined(__x86_64__)
460 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
461 * 0x40000000 is free */
463 addr
= (void *)0x40000000;
464 /* Cannot map more than that */
465 if (code_gen_buffer_size
> (800 * 1024 * 1024))
466 code_gen_buffer_size
= (800 * 1024 * 1024);
468 code_gen_buffer
= mmap(addr
, code_gen_buffer_size
,
469 PROT_WRITE
| PROT_READ
| PROT_EXEC
,
471 if (code_gen_buffer
== MAP_FAILED
) {
472 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
477 code_gen_buffer
= qemu_malloc(code_gen_buffer_size
);
478 map_exec(code_gen_buffer
, code_gen_buffer_size
);
480 #endif /* !USE_STATIC_CODE_GEN_BUFFER */
481 map_exec(code_gen_prologue
, sizeof(code_gen_prologue
));
482 code_gen_buffer_max_size
= code_gen_buffer_size
-
483 code_gen_max_block_size();
484 code_gen_max_blocks
= code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
485 tbs
= qemu_malloc(code_gen_max_blocks
* sizeof(TranslationBlock
));
488 /* Must be called before using the QEMU cpus. 'tb_size' is the size
489 (in bytes) allocated to the translation buffer. Zero means default
491 void cpu_exec_init_all(unsigned long tb_size
)
494 code_gen_alloc(tb_size
);
495 code_gen_ptr
= code_gen_buffer
;
497 #if !defined(CONFIG_USER_ONLY)
502 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
504 #define CPU_COMMON_SAVE_VERSION 1
506 static void cpu_common_save(QEMUFile
*f
, void *opaque
)
508 CPUState
*env
= opaque
;
510 qemu_put_be32s(f
, &env
->halted
);
511 qemu_put_be32s(f
, &env
->interrupt_request
);
514 static int cpu_common_load(QEMUFile
*f
, void *opaque
, int version_id
)
516 CPUState
*env
= opaque
;
518 if (version_id
!= CPU_COMMON_SAVE_VERSION
)
521 qemu_get_be32s(f
, &env
->halted
);
522 qemu_get_be32s(f
, &env
->interrupt_request
);
523 /* 0x01 was CPU_INTERRUPT_EXIT. This line can be removed when the
524 version_id is increased. */
525 env
->interrupt_request
&= ~0x01;
532 void cpu_exec_init(CPUState
*env
)
537 #if defined(CONFIG_USER_ONLY)
540 env
->next_cpu
= NULL
;
543 while (*penv
!= NULL
) {
544 penv
= (CPUState
**)&(*penv
)->next_cpu
;
547 env
->cpu_index
= cpu_index
;
548 TAILQ_INIT(&env
->breakpoints
);
549 TAILQ_INIT(&env
->watchpoints
);
551 #if defined(CONFIG_USER_ONLY)
554 #if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
555 register_savevm("cpu_common", cpu_index
, CPU_COMMON_SAVE_VERSION
,
556 cpu_common_save
, cpu_common_load
, env
);
557 register_savevm("cpu", cpu_index
, CPU_SAVE_VERSION
,
558 cpu_save
, cpu_load
, env
);
562 static inline void invalidate_page_bitmap(PageDesc
*p
)
564 if (p
->code_bitmap
) {
565 qemu_free(p
->code_bitmap
);
566 p
->code_bitmap
= NULL
;
568 p
->code_write_count
= 0;
571 /* set to NULL all the 'first_tb' fields in all PageDescs */
572 static void page_flush_tb(void)
577 for(i
= 0; i
< L1_SIZE
; i
++) {
580 for(j
= 0; j
< L2_SIZE
; j
++) {
582 invalidate_page_bitmap(p
);
589 /* flush all the translation blocks */
590 /* XXX: tb_flush is currently not thread safe */
591 void tb_flush(CPUState
*env1
)
594 #if defined(DEBUG_FLUSH)
595 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
596 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
598 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
600 if ((unsigned long)(code_gen_ptr
- code_gen_buffer
) > code_gen_buffer_size
)
601 cpu_abort(env1
, "Internal error: code buffer overflow\n");
605 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
606 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
609 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
612 code_gen_ptr
= code_gen_buffer
;
613 /* XXX: flush processor icache at this point if cache flush is
618 #ifdef DEBUG_TB_CHECK
620 static void tb_invalidate_check(target_ulong address
)
622 TranslationBlock
*tb
;
624 address
&= TARGET_PAGE_MASK
;
625 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
626 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
627 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
628 address
>= tb
->pc
+ tb
->size
)) {
629 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
630 address
, (long)tb
->pc
, tb
->size
);
636 /* verify that all the pages have correct rights for code */
637 static void tb_page_check(void)
639 TranslationBlock
*tb
;
640 int i
, flags1
, flags2
;
642 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
643 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
644 flags1
= page_get_flags(tb
->pc
);
645 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
646 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
647 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
648 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
654 static void tb_jmp_check(TranslationBlock
*tb
)
656 TranslationBlock
*tb1
;
659 /* suppress any remaining jumps to this TB */
663 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
666 tb1
= tb1
->jmp_next
[n1
];
668 /* check end of list */
670 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb
);
676 /* invalidate one TB */
677 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
680 TranslationBlock
*tb1
;
684 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
687 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
691 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
693 TranslationBlock
*tb1
;
699 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
701 *ptb
= tb1
->page_next
[n1
];
704 ptb
= &tb1
->page_next
[n1
];
708 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
710 TranslationBlock
*tb1
, **ptb
;
713 ptb
= &tb
->jmp_next
[n
];
716 /* find tb(n) in circular list */
720 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
721 if (n1
== n
&& tb1
== tb
)
724 ptb
= &tb1
->jmp_first
;
726 ptb
= &tb1
->jmp_next
[n1
];
729 /* now we can suppress tb(n) from the list */
730 *ptb
= tb
->jmp_next
[n
];
732 tb
->jmp_next
[n
] = NULL
;
736 /* reset the jump entry 'n' of a TB so that it is not chained to
738 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
740 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
743 void tb_phys_invalidate(TranslationBlock
*tb
, target_ulong page_addr
)
748 target_phys_addr_t phys_pc
;
749 TranslationBlock
*tb1
, *tb2
;
751 /* remove the TB from the hash list */
752 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
753 h
= tb_phys_hash_func(phys_pc
);
754 tb_remove(&tb_phys_hash
[h
], tb
,
755 offsetof(TranslationBlock
, phys_hash_next
));
757 /* remove the TB from the page list */
758 if (tb
->page_addr
[0] != page_addr
) {
759 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
760 tb_page_remove(&p
->first_tb
, tb
);
761 invalidate_page_bitmap(p
);
763 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
764 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
765 tb_page_remove(&p
->first_tb
, tb
);
766 invalidate_page_bitmap(p
);
769 tb_invalidated_flag
= 1;
771 /* remove the TB from the hash list */
772 h
= tb_jmp_cache_hash_func(tb
->pc
);
773 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
774 if (env
->tb_jmp_cache
[h
] == tb
)
775 env
->tb_jmp_cache
[h
] = NULL
;
778 /* suppress this TB from the two jump lists */
779 tb_jmp_remove(tb
, 0);
780 tb_jmp_remove(tb
, 1);
782 /* suppress any remaining jumps to this TB */
788 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
789 tb2
= tb1
->jmp_next
[n1
];
790 tb_reset_jump(tb1
, n1
);
791 tb1
->jmp_next
[n1
] = NULL
;
794 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
796 tb_phys_invalidate_count
++;
799 static inline void set_bits(uint8_t *tab
, int start
, int len
)
805 mask
= 0xff << (start
& 7);
806 if ((start
& ~7) == (end
& ~7)) {
808 mask
&= ~(0xff << (end
& 7));
813 start
= (start
+ 8) & ~7;
815 while (start
< end1
) {
820 mask
= ~(0xff << (end
& 7));
826 static void build_page_bitmap(PageDesc
*p
)
828 int n
, tb_start
, tb_end
;
829 TranslationBlock
*tb
;
831 p
->code_bitmap
= qemu_mallocz(TARGET_PAGE_SIZE
/ 8);
836 tb
= (TranslationBlock
*)((long)tb
& ~3);
837 /* NOTE: this is subtle as a TB may span two physical pages */
839 /* NOTE: tb_end may be after the end of the page, but
840 it is not a problem */
841 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
842 tb_end
= tb_start
+ tb
->size
;
843 if (tb_end
> TARGET_PAGE_SIZE
)
844 tb_end
= TARGET_PAGE_SIZE
;
847 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
849 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
850 tb
= tb
->page_next
[n
];
854 TranslationBlock
*tb_gen_code(CPUState
*env
,
855 target_ulong pc
, target_ulong cs_base
,
856 int flags
, int cflags
)
858 TranslationBlock
*tb
;
860 target_ulong phys_pc
, phys_page2
, virt_page2
;
863 phys_pc
= get_phys_addr_code(env
, pc
);
866 /* flush must be done */
868 /* cannot fail at this point */
870 /* Don't forget to invalidate previous TB info. */
871 tb_invalidated_flag
= 1;
873 tc_ptr
= code_gen_ptr
;
875 tb
->cs_base
= cs_base
;
878 cpu_gen_code(env
, tb
, &code_gen_size
);
879 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
881 /* check next page if needed */
882 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
884 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
885 phys_page2
= get_phys_addr_code(env
, virt_page2
);
887 tb_link_phys(tb
, phys_pc
, phys_page2
);
891 /* invalidate all TBs which intersect with the target physical page
892 starting in range [start;end[. NOTE: start and end must refer to
893 the same physical page. 'is_cpu_write_access' should be true if called
894 from a real cpu write access: the virtual CPU will exit the current
895 TB if code is modified inside this TB. */
896 void tb_invalidate_phys_page_range(target_phys_addr_t start
, target_phys_addr_t end
,
897 int is_cpu_write_access
)
899 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
900 CPUState
*env
= cpu_single_env
;
901 target_ulong tb_start
, tb_end
;
904 #ifdef TARGET_HAS_PRECISE_SMC
905 int current_tb_not_found
= is_cpu_write_access
;
906 TranslationBlock
*current_tb
= NULL
;
907 int current_tb_modified
= 0;
908 target_ulong current_pc
= 0;
909 target_ulong current_cs_base
= 0;
910 int current_flags
= 0;
911 #endif /* TARGET_HAS_PRECISE_SMC */
913 p
= page_find(start
>> TARGET_PAGE_BITS
);
916 if (!p
->code_bitmap
&&
917 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
918 is_cpu_write_access
) {
919 /* build code bitmap */
920 build_page_bitmap(p
);
923 /* we remove all the TBs in the range [start, end[ */
924 /* XXX: see if in some cases it could be faster to invalidate all the code */
928 tb
= (TranslationBlock
*)((long)tb
& ~3);
929 tb_next
= tb
->page_next
[n
];
930 /* NOTE: this is subtle as a TB may span two physical pages */
932 /* NOTE: tb_end may be after the end of the page, but
933 it is not a problem */
934 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
935 tb_end
= tb_start
+ tb
->size
;
937 tb_start
= tb
->page_addr
[1];
938 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
940 if (!(tb_end
<= start
|| tb_start
>= end
)) {
941 #ifdef TARGET_HAS_PRECISE_SMC
942 if (current_tb_not_found
) {
943 current_tb_not_found
= 0;
945 if (env
->mem_io_pc
) {
946 /* now we have a real cpu fault */
947 current_tb
= tb_find_pc(env
->mem_io_pc
);
950 if (current_tb
== tb
&&
951 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
952 /* If we are modifying the current TB, we must stop
953 its execution. We could be more precise by checking
954 that the modification is after the current PC, but it
955 would require a specialized function to partially
956 restore the CPU state */
958 current_tb_modified
= 1;
959 cpu_restore_state(current_tb
, env
,
960 env
->mem_io_pc
, NULL
);
961 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
964 #endif /* TARGET_HAS_PRECISE_SMC */
965 /* we need to do that to handle the case where a signal
966 occurs while doing tb_phys_invalidate() */
969 saved_tb
= env
->current_tb
;
970 env
->current_tb
= NULL
;
972 tb_phys_invalidate(tb
, -1);
974 env
->current_tb
= saved_tb
;
975 if (env
->interrupt_request
&& env
->current_tb
)
976 cpu_interrupt(env
, env
->interrupt_request
);
981 #if !defined(CONFIG_USER_ONLY)
982 /* if no code remaining, no need to continue to use slow writes */
984 invalidate_page_bitmap(p
);
985 if (is_cpu_write_access
) {
986 tlb_unprotect_code_phys(env
, start
, env
->mem_io_vaddr
);
990 #ifdef TARGET_HAS_PRECISE_SMC
991 if (current_tb_modified
) {
992 /* we generate a block containing just the instruction
993 modifying the memory. It will ensure that it cannot modify
995 env
->current_tb
= NULL
;
996 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
997 cpu_resume_from_signal(env
, NULL
);
1002 /* len must be <= 8 and start must be a multiple of len */
1003 static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start
, int len
)
1009 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1010 cpu_single_env
->mem_io_vaddr
, len
,
1011 cpu_single_env
->eip
,
1012 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
1015 p
= page_find(start
>> TARGET_PAGE_BITS
);
1018 if (p
->code_bitmap
) {
1019 offset
= start
& ~TARGET_PAGE_MASK
;
1020 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1021 if (b
& ((1 << len
) - 1))
1025 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1029 #if !defined(CONFIG_SOFTMMU)
1030 static void tb_invalidate_phys_page(target_phys_addr_t addr
,
1031 unsigned long pc
, void *puc
)
1033 TranslationBlock
*tb
;
1036 #ifdef TARGET_HAS_PRECISE_SMC
1037 TranslationBlock
*current_tb
= NULL
;
1038 CPUState
*env
= cpu_single_env
;
1039 int current_tb_modified
= 0;
1040 target_ulong current_pc
= 0;
1041 target_ulong current_cs_base
= 0;
1042 int current_flags
= 0;
1045 addr
&= TARGET_PAGE_MASK
;
1046 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1050 #ifdef TARGET_HAS_PRECISE_SMC
1051 if (tb
&& pc
!= 0) {
1052 current_tb
= tb_find_pc(pc
);
1055 while (tb
!= NULL
) {
1057 tb
= (TranslationBlock
*)((long)tb
& ~3);
1058 #ifdef TARGET_HAS_PRECISE_SMC
1059 if (current_tb
== tb
&&
1060 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1061 /* If we are modifying the current TB, we must stop
1062 its execution. We could be more precise by checking
1063 that the modification is after the current PC, but it
1064 would require a specialized function to partially
1065 restore the CPU state */
1067 current_tb_modified
= 1;
1068 cpu_restore_state(current_tb
, env
, pc
, puc
);
1069 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1072 #endif /* TARGET_HAS_PRECISE_SMC */
1073 tb_phys_invalidate(tb
, addr
);
1074 tb
= tb
->page_next
[n
];
1077 #ifdef TARGET_HAS_PRECISE_SMC
1078 if (current_tb_modified
) {
1079 /* we generate a block containing just the instruction
1080 modifying the memory. It will ensure that it cannot modify
1082 env
->current_tb
= NULL
;
1083 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1084 cpu_resume_from_signal(env
, puc
);
1090 /* add the tb in the target page and protect it if necessary */
1091 static inline void tb_alloc_page(TranslationBlock
*tb
,
1092 unsigned int n
, target_ulong page_addr
)
1095 TranslationBlock
*last_first_tb
;
1097 tb
->page_addr
[n
] = page_addr
;
1098 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
);
1099 tb
->page_next
[n
] = p
->first_tb
;
1100 last_first_tb
= p
->first_tb
;
1101 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
1102 invalidate_page_bitmap(p
);
1104 #if defined(TARGET_HAS_SMC) || 1
1106 #if defined(CONFIG_USER_ONLY)
1107 if (p
->flags
& PAGE_WRITE
) {
1112 /* force the host page as non writable (writes will have a
1113 page fault + mprotect overhead) */
1114 page_addr
&= qemu_host_page_mask
;
1116 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1117 addr
+= TARGET_PAGE_SIZE
) {
1119 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
1123 p2
->flags
&= ~PAGE_WRITE
;
1124 page_get_flags(addr
);
1126 mprotect(g2h(page_addr
), qemu_host_page_size
,
1127 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1128 #ifdef DEBUG_TB_INVALIDATE
1129 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1134 /* if some code is already present, then the pages are already
1135 protected. So we handle the case where only the first TB is
1136 allocated in a physical page */
1137 if (!last_first_tb
) {
1138 tlb_protect_code(page_addr
);
1142 #endif /* TARGET_HAS_SMC */
1145 /* Allocate a new translation block. Flush the translation buffer if
1146 too many translation blocks or too much generated code. */
1147 TranslationBlock
*tb_alloc(target_ulong pc
)
1149 TranslationBlock
*tb
;
1151 if (nb_tbs
>= code_gen_max_blocks
||
1152 (code_gen_ptr
- code_gen_buffer
) >= code_gen_buffer_max_size
)
1154 tb
= &tbs
[nb_tbs
++];
1160 void tb_free(TranslationBlock
*tb
)
1162 /* In practice this is mostly used for single use temporary TB
1163 Ignore the hard cases and just back up if this TB happens to
1164 be the last one generated. */
1165 if (nb_tbs
> 0 && tb
== &tbs
[nb_tbs
- 1]) {
1166 code_gen_ptr
= tb
->tc_ptr
;
1171 /* add a new TB and link it to the physical page tables. phys_page2 is
1172 (-1) to indicate that only one page contains the TB. */
1173 void tb_link_phys(TranslationBlock
*tb
,
1174 target_ulong phys_pc
, target_ulong phys_page2
)
1177 TranslationBlock
**ptb
;
1179 /* Grab the mmap lock to stop another thread invalidating this TB
1180 before we are done. */
1182 /* add in the physical hash table */
1183 h
= tb_phys_hash_func(phys_pc
);
1184 ptb
= &tb_phys_hash
[h
];
1185 tb
->phys_hash_next
= *ptb
;
1188 /* add in the page list */
1189 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1190 if (phys_page2
!= -1)
1191 tb_alloc_page(tb
, 1, phys_page2
);
1193 tb
->page_addr
[1] = -1;
1195 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
1196 tb
->jmp_next
[0] = NULL
;
1197 tb
->jmp_next
[1] = NULL
;
1199 /* init original jump addresses */
1200 if (tb
->tb_next_offset
[0] != 0xffff)
1201 tb_reset_jump(tb
, 0);
1202 if (tb
->tb_next_offset
[1] != 0xffff)
1203 tb_reset_jump(tb
, 1);
1205 #ifdef DEBUG_TB_CHECK
1211 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1212 tb[1].tc_ptr. Return NULL if not found */
1213 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1215 int m_min
, m_max
, m
;
1217 TranslationBlock
*tb
;
1221 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1222 tc_ptr
>= (unsigned long)code_gen_ptr
)
1224 /* binary search (cf Knuth) */
1227 while (m_min
<= m_max
) {
1228 m
= (m_min
+ m_max
) >> 1;
1230 v
= (unsigned long)tb
->tc_ptr
;
1233 else if (tc_ptr
< v
) {
1242 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1244 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1246 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1249 tb1
= tb
->jmp_next
[n
];
1251 /* find head of list */
1254 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1257 tb1
= tb1
->jmp_next
[n1
];
1259 /* we are now sure now that tb jumps to tb1 */
1262 /* remove tb from the jmp_first list */
1263 ptb
= &tb_next
->jmp_first
;
1267 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1268 if (n1
== n
&& tb1
== tb
)
1270 ptb
= &tb1
->jmp_next
[n1
];
1272 *ptb
= tb
->jmp_next
[n
];
1273 tb
->jmp_next
[n
] = NULL
;
1275 /* suppress the jump to next tb in generated code */
1276 tb_reset_jump(tb
, n
);
1278 /* suppress jumps in the tb on which we could have jumped */
1279 tb_reset_jump_recursive(tb_next
);
1283 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1285 tb_reset_jump_recursive2(tb
, 0);
1286 tb_reset_jump_recursive2(tb
, 1);
1289 #if defined(TARGET_HAS_ICE)
1290 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1292 target_phys_addr_t addr
;
1294 ram_addr_t ram_addr
;
1297 addr
= cpu_get_phys_page_debug(env
, pc
);
1298 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1300 pd
= IO_MEM_UNASSIGNED
;
1302 pd
= p
->phys_offset
;
1304 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1305 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1309 /* Add a watchpoint. */
1310 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
, target_ulong len
,
1311 int flags
, CPUWatchpoint
**watchpoint
)
1313 target_ulong len_mask
= ~(len
- 1);
1316 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1317 if ((len
!= 1 && len
!= 2 && len
!= 4 && len
!= 8) || (addr
& ~len_mask
)) {
1318 fprintf(stderr
, "qemu: tried to set invalid watchpoint at "
1319 TARGET_FMT_lx
", len=" TARGET_FMT_lu
"\n", addr
, len
);
1322 wp
= qemu_malloc(sizeof(*wp
));
1325 wp
->len_mask
= len_mask
;
1328 /* keep all GDB-injected watchpoints in front */
1330 TAILQ_INSERT_HEAD(&env
->watchpoints
, wp
, entry
);
1332 TAILQ_INSERT_TAIL(&env
->watchpoints
, wp
, entry
);
1334 tlb_flush_page(env
, addr
);
1341 /* Remove a specific watchpoint. */
1342 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
, target_ulong len
,
1345 target_ulong len_mask
= ~(len
- 1);
1348 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1349 if (addr
== wp
->vaddr
&& len_mask
== wp
->len_mask
1350 && flags
== (wp
->flags
& ~BP_WATCHPOINT_HIT
)) {
1351 cpu_watchpoint_remove_by_ref(env
, wp
);
1358 /* Remove a specific watchpoint by reference. */
1359 void cpu_watchpoint_remove_by_ref(CPUState
*env
, CPUWatchpoint
*watchpoint
)
1361 TAILQ_REMOVE(&env
->watchpoints
, watchpoint
, entry
);
1363 tlb_flush_page(env
, watchpoint
->vaddr
);
1365 qemu_free(watchpoint
);
1368 /* Remove all matching watchpoints. */
1369 void cpu_watchpoint_remove_all(CPUState
*env
, int mask
)
1371 CPUWatchpoint
*wp
, *next
;
1373 TAILQ_FOREACH_SAFE(wp
, &env
->watchpoints
, entry
, next
) {
1374 if (wp
->flags
& mask
)
1375 cpu_watchpoint_remove_by_ref(env
, wp
);
1379 /* Add a breakpoint. */
1380 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
, int flags
,
1381 CPUBreakpoint
**breakpoint
)
1383 #if defined(TARGET_HAS_ICE)
1386 bp
= qemu_malloc(sizeof(*bp
));
1391 /* keep all GDB-injected breakpoints in front */
1393 TAILQ_INSERT_HEAD(&env
->breakpoints
, bp
, entry
);
1395 TAILQ_INSERT_TAIL(&env
->breakpoints
, bp
, entry
);
1397 breakpoint_invalidate(env
, pc
);
1407 /* Remove a specific breakpoint. */
1408 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
, int flags
)
1410 #if defined(TARGET_HAS_ICE)
1413 TAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1414 if (bp
->pc
== pc
&& bp
->flags
== flags
) {
1415 cpu_breakpoint_remove_by_ref(env
, bp
);
1425 /* Remove a specific breakpoint by reference. */
1426 void cpu_breakpoint_remove_by_ref(CPUState
*env
, CPUBreakpoint
*breakpoint
)
1428 #if defined(TARGET_HAS_ICE)
1429 TAILQ_REMOVE(&env
->breakpoints
, breakpoint
, entry
);
1431 breakpoint_invalidate(env
, breakpoint
->pc
);
1433 qemu_free(breakpoint
);
1437 /* Remove all matching breakpoints. */
1438 void cpu_breakpoint_remove_all(CPUState
*env
, int mask
)
1440 #if defined(TARGET_HAS_ICE)
1441 CPUBreakpoint
*bp
, *next
;
1443 TAILQ_FOREACH_SAFE(bp
, &env
->breakpoints
, entry
, next
) {
1444 if (bp
->flags
& mask
)
1445 cpu_breakpoint_remove_by_ref(env
, bp
);
1450 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1451 CPU loop after each instruction */
1452 void cpu_single_step(CPUState
*env
, int enabled
)
1454 #if defined(TARGET_HAS_ICE)
1455 if (env
->singlestep_enabled
!= enabled
) {
1456 env
->singlestep_enabled
= enabled
;
1458 kvm_update_guest_debug(env
, 0);
1460 /* must flush all the translated code to avoid inconsistancies */
1461 /* XXX: only flush what is necessary */
1468 /* enable or disable low levels log */
1469 void cpu_set_log(int log_flags
)
1471 loglevel
= log_flags
;
1472 if (loglevel
&& !logfile
) {
1473 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1475 perror(logfilename
);
1478 #if !defined(CONFIG_SOFTMMU)
1479 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1481 static char logfile_buf
[4096];
1482 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1485 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1489 if (!loglevel
&& logfile
) {
1495 void cpu_set_log_filename(const char *filename
)
1497 logfilename
= strdup(filename
);
1502 cpu_set_log(loglevel
);
1505 static void cpu_unlink_tb(CPUState
*env
)
1507 #if defined(USE_NPTL)
1508 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1509 problem and hope the cpu will stop of its own accord. For userspace
1510 emulation this often isn't actually as bad as it sounds. Often
1511 signals are used primarily to interrupt blocking syscalls. */
1513 TranslationBlock
*tb
;
1514 static spinlock_t interrupt_lock
= SPIN_LOCK_UNLOCKED
;
1516 tb
= env
->current_tb
;
1517 /* if the cpu is currently executing code, we must unlink it and
1518 all the potentially executing TB */
1519 if (tb
&& !testandset(&interrupt_lock
)) {
1520 env
->current_tb
= NULL
;
1521 tb_reset_jump_recursive(tb
);
1522 resetlock(&interrupt_lock
);
1527 /* mask must never be zero, except for A20 change call */
1528 void cpu_interrupt(CPUState
*env
, int mask
)
1532 old_mask
= env
->interrupt_request
;
1533 env
->interrupt_request
|= mask
;
1536 env
->icount_decr
.u16
.high
= 0xffff;
1537 #ifndef CONFIG_USER_ONLY
1539 && (mask
& ~old_mask
) != 0) {
1540 cpu_abort(env
, "Raised interrupt while not in I/O function");
1548 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1550 env
->interrupt_request
&= ~mask
;
1553 void cpu_exit(CPUState
*env
)
1555 env
->exit_request
= 1;
1559 const CPULogItem cpu_log_items
[] = {
1560 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1561 "show generated host assembly code for each compiled TB" },
1562 { CPU_LOG_TB_IN_ASM
, "in_asm",
1563 "show target assembly code for each compiled TB" },
1564 { CPU_LOG_TB_OP
, "op",
1565 "show micro ops for each compiled TB" },
1566 { CPU_LOG_TB_OP_OPT
, "op_opt",
1569 "before eflags optimization and "
1571 "after liveness analysis" },
1572 { CPU_LOG_INT
, "int",
1573 "show interrupts/exceptions in short format" },
1574 { CPU_LOG_EXEC
, "exec",
1575 "show trace before each executed TB (lots of logs)" },
1576 { CPU_LOG_TB_CPU
, "cpu",
1577 "show CPU state before block translation" },
1579 { CPU_LOG_PCALL
, "pcall",
1580 "show protected mode far calls/returns/exceptions" },
1581 { CPU_LOG_RESET
, "cpu_reset",
1582 "show CPU state before CPU resets" },
1585 { CPU_LOG_IOPORT
, "ioport",
1586 "show all i/o ports accesses" },
1591 static int cmp1(const char *s1
, int n
, const char *s2
)
1593 if (strlen(s2
) != n
)
1595 return memcmp(s1
, s2
, n
) == 0;
1598 /* takes a comma separated list of log masks. Return 0 if error. */
1599 int cpu_str_to_log_mask(const char *str
)
1601 const CPULogItem
*item
;
1608 p1
= strchr(p
, ',');
1611 if(cmp1(p
,p1
-p
,"all")) {
1612 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1616 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1617 if (cmp1(p
, p1
- p
, item
->name
))
1631 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1638 fprintf(stderr
, "qemu: fatal: ");
1639 vfprintf(stderr
, fmt
, ap
);
1640 fprintf(stderr
, "\n");
1642 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1644 cpu_dump_state(env
, stderr
, fprintf
, 0);
1646 if (qemu_log_enabled()) {
1647 qemu_log("qemu: fatal: ");
1648 qemu_log_vprintf(fmt
, ap2
);
1651 log_cpu_state(env
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1653 log_cpu_state(env
, 0);
1663 CPUState
*cpu_copy(CPUState
*env
)
1665 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1666 CPUState
*next_cpu
= new_env
->next_cpu
;
1667 int cpu_index
= new_env
->cpu_index
;
1668 #if defined(TARGET_HAS_ICE)
1673 memcpy(new_env
, env
, sizeof(CPUState
));
1675 /* Preserve chaining and index. */
1676 new_env
->next_cpu
= next_cpu
;
1677 new_env
->cpu_index
= cpu_index
;
1679 /* Clone all break/watchpoints.
1680 Note: Once we support ptrace with hw-debug register access, make sure
1681 BP_CPU break/watchpoints are handled correctly on clone. */
1682 TAILQ_INIT(&env
->breakpoints
);
1683 TAILQ_INIT(&env
->watchpoints
);
1684 #if defined(TARGET_HAS_ICE)
1685 TAILQ_FOREACH(bp
, &env
->breakpoints
, entry
) {
1686 cpu_breakpoint_insert(new_env
, bp
->pc
, bp
->flags
, NULL
);
1688 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
1689 cpu_watchpoint_insert(new_env
, wp
->vaddr
, (~wp
->len_mask
) + 1,
1697 #if !defined(CONFIG_USER_ONLY)
1699 static inline void tlb_flush_jmp_cache(CPUState
*env
, target_ulong addr
)
1703 /* Discard jump cache entries for any tb which might potentially
1704 overlap the flushed page. */
1705 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1706 memset (&env
->tb_jmp_cache
[i
], 0,
1707 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1709 i
= tb_jmp_cache_hash_page(addr
);
1710 memset (&env
->tb_jmp_cache
[i
], 0,
1711 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1714 /* NOTE: if flush_global is true, also flush global entries (not
1716 void tlb_flush(CPUState
*env
, int flush_global
)
1720 #if defined(DEBUG_TLB)
1721 printf("tlb_flush:\n");
1723 /* must reset current TB so that interrupts cannot modify the
1724 links while we are modifying them */
1725 env
->current_tb
= NULL
;
1727 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1728 env
->tlb_table
[0][i
].addr_read
= -1;
1729 env
->tlb_table
[0][i
].addr_write
= -1;
1730 env
->tlb_table
[0][i
].addr_code
= -1;
1731 env
->tlb_table
[1][i
].addr_read
= -1;
1732 env
->tlb_table
[1][i
].addr_write
= -1;
1733 env
->tlb_table
[1][i
].addr_code
= -1;
1734 #if (NB_MMU_MODES >= 3)
1735 env
->tlb_table
[2][i
].addr_read
= -1;
1736 env
->tlb_table
[2][i
].addr_write
= -1;
1737 env
->tlb_table
[2][i
].addr_code
= -1;
1738 #if (NB_MMU_MODES == 4)
1739 env
->tlb_table
[3][i
].addr_read
= -1;
1740 env
->tlb_table
[3][i
].addr_write
= -1;
1741 env
->tlb_table
[3][i
].addr_code
= -1;
1746 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1749 if (env
->kqemu_enabled
) {
1750 kqemu_flush(env
, flush_global
);
1756 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1758 if (addr
== (tlb_entry
->addr_read
&
1759 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1760 addr
== (tlb_entry
->addr_write
&
1761 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1762 addr
== (tlb_entry
->addr_code
&
1763 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1764 tlb_entry
->addr_read
= -1;
1765 tlb_entry
->addr_write
= -1;
1766 tlb_entry
->addr_code
= -1;
1770 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1774 #if defined(DEBUG_TLB)
1775 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1777 /* must reset current TB so that interrupts cannot modify the
1778 links while we are modifying them */
1779 env
->current_tb
= NULL
;
1781 addr
&= TARGET_PAGE_MASK
;
1782 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1783 tlb_flush_entry(&env
->tlb_table
[0][i
], addr
);
1784 tlb_flush_entry(&env
->tlb_table
[1][i
], addr
);
1785 #if (NB_MMU_MODES >= 3)
1786 tlb_flush_entry(&env
->tlb_table
[2][i
], addr
);
1787 #if (NB_MMU_MODES == 4)
1788 tlb_flush_entry(&env
->tlb_table
[3][i
], addr
);
1792 tlb_flush_jmp_cache(env
, addr
);
1795 if (env
->kqemu_enabled
) {
1796 kqemu_flush_page(env
, addr
);
1801 /* update the TLBs so that writes to code in the virtual page 'addr'
1803 static void tlb_protect_code(ram_addr_t ram_addr
)
1805 cpu_physical_memory_reset_dirty(ram_addr
,
1806 ram_addr
+ TARGET_PAGE_SIZE
,
1810 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1811 tested for self modifying code */
1812 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1815 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
1818 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1819 unsigned long start
, unsigned long length
)
1822 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1823 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1824 if ((addr
- start
) < length
) {
1825 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | TLB_NOTDIRTY
;
1830 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1834 unsigned long length
, start1
;
1838 start
&= TARGET_PAGE_MASK
;
1839 end
= TARGET_PAGE_ALIGN(end
);
1841 length
= end
- start
;
1844 len
= length
>> TARGET_PAGE_BITS
;
1846 /* XXX: should not depend on cpu context */
1848 if (env
->kqemu_enabled
) {
1851 for(i
= 0; i
< len
; i
++) {
1852 kqemu_set_notdirty(env
, addr
);
1853 addr
+= TARGET_PAGE_SIZE
;
1857 mask
= ~dirty_flags
;
1858 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
1859 for(i
= 0; i
< len
; i
++)
1862 /* we modify the TLB cache so that the dirty bit will be set again
1863 when accessing the range */
1864 start1
= start
+ (unsigned long)phys_ram_base
;
1865 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1866 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1867 tlb_reset_dirty_range(&env
->tlb_table
[0][i
], start1
, length
);
1868 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1869 tlb_reset_dirty_range(&env
->tlb_table
[1][i
], start1
, length
);
1870 #if (NB_MMU_MODES >= 3)
1871 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1872 tlb_reset_dirty_range(&env
->tlb_table
[2][i
], start1
, length
);
1873 #if (NB_MMU_MODES == 4)
1874 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1875 tlb_reset_dirty_range(&env
->tlb_table
[3][i
], start1
, length
);
1881 int cpu_physical_memory_set_dirty_tracking(int enable
)
1883 in_migration
= enable
;
1887 int cpu_physical_memory_get_dirty_tracking(void)
1889 return in_migration
;
1892 void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr
, target_phys_addr_t end_addr
)
1895 kvm_physical_sync_dirty_bitmap(start_addr
, end_addr
);
1898 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
1900 ram_addr_t ram_addr
;
1902 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1903 ram_addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) +
1904 tlb_entry
->addend
- (unsigned long)phys_ram_base
;
1905 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
1906 tlb_entry
->addr_write
|= TLB_NOTDIRTY
;
1911 /* update the TLB according to the current state of the dirty bits */
1912 void cpu_tlb_update_dirty(CPUState
*env
)
1915 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1916 tlb_update_dirty(&env
->tlb_table
[0][i
]);
1917 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1918 tlb_update_dirty(&env
->tlb_table
[1][i
]);
1919 #if (NB_MMU_MODES >= 3)
1920 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1921 tlb_update_dirty(&env
->tlb_table
[2][i
]);
1922 #if (NB_MMU_MODES == 4)
1923 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1924 tlb_update_dirty(&env
->tlb_table
[3][i
]);
1929 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
, target_ulong vaddr
)
1931 if (tlb_entry
->addr_write
== (vaddr
| TLB_NOTDIRTY
))
1932 tlb_entry
->addr_write
= vaddr
;
1935 /* update the TLB corresponding to virtual page vaddr
1936 so that it is no longer dirty */
1937 static inline void tlb_set_dirty(CPUState
*env
, target_ulong vaddr
)
1941 vaddr
&= TARGET_PAGE_MASK
;
1942 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1943 tlb_set_dirty1(&env
->tlb_table
[0][i
], vaddr
);
1944 tlb_set_dirty1(&env
->tlb_table
[1][i
], vaddr
);
1945 #if (NB_MMU_MODES >= 3)
1946 tlb_set_dirty1(&env
->tlb_table
[2][i
], vaddr
);
1947 #if (NB_MMU_MODES == 4)
1948 tlb_set_dirty1(&env
->tlb_table
[3][i
], vaddr
);
1953 /* add a new TLB entry. At most one entry for a given virtual address
1954 is permitted. Return 0 if OK or 2 if the page could not be mapped
1955 (can only happen in non SOFTMMU mode for I/O pages or pages
1956 conflicting with the host address space). */
1957 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
1958 target_phys_addr_t paddr
, int prot
,
1959 int mmu_idx
, int is_softmmu
)
1964 target_ulong address
;
1965 target_ulong code_address
;
1966 target_phys_addr_t addend
;
1970 target_phys_addr_t iotlb
;
1972 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
1974 pd
= IO_MEM_UNASSIGNED
;
1976 pd
= p
->phys_offset
;
1978 #if defined(DEBUG_TLB)
1979 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1980 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
1985 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
1986 /* IO memory case (romd handled later) */
1987 address
|= TLB_MMIO
;
1989 addend
= (unsigned long)phys_ram_base
+ (pd
& TARGET_PAGE_MASK
);
1990 if ((pd
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
) {
1992 iotlb
= pd
& TARGET_PAGE_MASK
;
1993 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
)
1994 iotlb
|= IO_MEM_NOTDIRTY
;
1996 iotlb
|= IO_MEM_ROM
;
1998 /* IO handlers are currently passed a phsical address.
1999 It would be nice to pass an offset from the base address
2000 of that region. This would avoid having to special case RAM,
2001 and avoid full address decoding in every device.
2002 We can't use the high bits of pd for this because
2003 IO_MEM_ROMD uses these as a ram address. */
2004 iotlb
= (pd
& ~TARGET_PAGE_MASK
);
2006 iotlb
+= p
->region_offset
;
2012 code_address
= address
;
2013 /* Make accesses to pages with watchpoints go via the
2014 watchpoint trap routines. */
2015 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2016 if (vaddr
== (wp
->vaddr
& TARGET_PAGE_MASK
)) {
2017 iotlb
= io_mem_watch
+ paddr
;
2018 /* TODO: The memory case can be optimized by not trapping
2019 reads of pages with a write breakpoint. */
2020 address
|= TLB_MMIO
;
2024 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
2025 env
->iotlb
[mmu_idx
][index
] = iotlb
- vaddr
;
2026 te
= &env
->tlb_table
[mmu_idx
][index
];
2027 te
->addend
= addend
- vaddr
;
2028 if (prot
& PAGE_READ
) {
2029 te
->addr_read
= address
;
2034 if (prot
& PAGE_EXEC
) {
2035 te
->addr_code
= code_address
;
2039 if (prot
& PAGE_WRITE
) {
2040 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
2041 (pd
& IO_MEM_ROMD
)) {
2042 /* Write access calls the I/O callback. */
2043 te
->addr_write
= address
| TLB_MMIO
;
2044 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
2045 !cpu_physical_memory_is_dirty(pd
)) {
2046 te
->addr_write
= address
| TLB_NOTDIRTY
;
2048 te
->addr_write
= address
;
2051 te
->addr_write
= -1;
2058 void tlb_flush(CPUState
*env
, int flush_global
)
2062 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
2066 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
2067 target_phys_addr_t paddr
, int prot
,
2068 int mmu_idx
, int is_softmmu
)
2073 /* dump memory mappings */
2074 void page_dump(FILE *f
)
2076 unsigned long start
, end
;
2077 int i
, j
, prot
, prot1
;
2080 fprintf(f
, "%-8s %-8s %-8s %s\n",
2081 "start", "end", "size", "prot");
2085 for(i
= 0; i
<= L1_SIZE
; i
++) {
2090 for(j
= 0;j
< L2_SIZE
; j
++) {
2095 if (prot1
!= prot
) {
2096 end
= (i
<< (32 - L1_BITS
)) | (j
<< TARGET_PAGE_BITS
);
2098 fprintf(f
, "%08lx-%08lx %08lx %c%c%c\n",
2099 start
, end
, end
- start
,
2100 prot
& PAGE_READ
? 'r' : '-',
2101 prot
& PAGE_WRITE
? 'w' : '-',
2102 prot
& PAGE_EXEC
? 'x' : '-');
2116 int page_get_flags(target_ulong address
)
2120 p
= page_find(address
>> TARGET_PAGE_BITS
);
2126 /* modify the flags of a page and invalidate the code if
2127 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2128 depending on PAGE_WRITE */
2129 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2134 /* mmap_lock should already be held. */
2135 start
= start
& TARGET_PAGE_MASK
;
2136 end
= TARGET_PAGE_ALIGN(end
);
2137 if (flags
& PAGE_WRITE
)
2138 flags
|= PAGE_WRITE_ORG
;
2139 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2140 p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
);
2141 /* We may be called for host regions that are outside guest
2145 /* if the write protection is set, then we invalidate the code
2147 if (!(p
->flags
& PAGE_WRITE
) &&
2148 (flags
& PAGE_WRITE
) &&
2150 tb_invalidate_phys_page(addr
, 0, NULL
);
2156 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2162 if (start
+ len
< start
)
2163 /* we've wrapped around */
2166 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
2167 start
= start
& TARGET_PAGE_MASK
;
2169 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
2170 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2173 if( !(p
->flags
& PAGE_VALID
) )
2176 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
2178 if (flags
& PAGE_WRITE
) {
2179 if (!(p
->flags
& PAGE_WRITE_ORG
))
2181 /* unprotect the page if it was put read-only because it
2182 contains translated code */
2183 if (!(p
->flags
& PAGE_WRITE
)) {
2184 if (!page_unprotect(addr
, 0, NULL
))
2193 /* called from signal handler: invalidate the code and unprotect the
2194 page. Return TRUE if the fault was succesfully handled. */
2195 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
2197 unsigned int page_index
, prot
, pindex
;
2199 target_ulong host_start
, host_end
, addr
;
2201 /* Technically this isn't safe inside a signal handler. However we
2202 know this only ever happens in a synchronous SEGV handler, so in
2203 practice it seems to be ok. */
2206 host_start
= address
& qemu_host_page_mask
;
2207 page_index
= host_start
>> TARGET_PAGE_BITS
;
2208 p1
= page_find(page_index
);
2213 host_end
= host_start
+ qemu_host_page_size
;
2216 for(addr
= host_start
;addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2220 /* if the page was really writable, then we change its
2221 protection back to writable */
2222 if (prot
& PAGE_WRITE_ORG
) {
2223 pindex
= (address
- host_start
) >> TARGET_PAGE_BITS
;
2224 if (!(p1
[pindex
].flags
& PAGE_WRITE
)) {
2225 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2226 (prot
& PAGE_BITS
) | PAGE_WRITE
);
2227 p1
[pindex
].flags
|= PAGE_WRITE
;
2228 /* and since the content will be modified, we must invalidate
2229 the corresponding translated code. */
2230 tb_invalidate_phys_page(address
, pc
, puc
);
2231 #ifdef DEBUG_TB_CHECK
2232 tb_invalidate_check(address
);
2242 static inline void tlb_set_dirty(CPUState
*env
,
2243 unsigned long addr
, target_ulong vaddr
)
2246 #endif /* defined(CONFIG_USER_ONLY) */
2248 #if !defined(CONFIG_USER_ONLY)
2250 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2251 ram_addr_t memory
, ram_addr_t region_offset
);
2252 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2253 ram_addr_t orig_memory
, ram_addr_t region_offset
);
2254 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2257 if (addr > start_addr) \
2260 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2261 if (start_addr2 > 0) \
2265 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2266 end_addr2 = TARGET_PAGE_SIZE - 1; \
2268 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2269 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2274 /* register physical memory. 'size' must be a multiple of the target
2275 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2276 io memory page. The address used when calling the IO function is
2277 the offset from the start of the region, plus region_offset. Both
2278 start_region and regon_offset are rounded down to a page boundary
2279 before calculating this offset. This should not be a problem unless
2280 the low bits of start_addr and region_offset differ. */
2281 void cpu_register_physical_memory_offset(target_phys_addr_t start_addr
,
2283 ram_addr_t phys_offset
,
2284 ram_addr_t region_offset
)
2286 target_phys_addr_t addr
, end_addr
;
2289 ram_addr_t orig_size
= size
;
2293 /* XXX: should not depend on cpu context */
2295 if (env
->kqemu_enabled
) {
2296 kqemu_set_phys_mem(start_addr
, size
, phys_offset
);
2300 kvm_set_phys_mem(start_addr
, size
, phys_offset
);
2302 if (phys_offset
== IO_MEM_UNASSIGNED
) {
2303 region_offset
= start_addr
;
2305 region_offset
&= TARGET_PAGE_MASK
;
2306 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2307 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2308 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2309 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2310 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2311 ram_addr_t orig_memory
= p
->phys_offset
;
2312 target_phys_addr_t start_addr2
, end_addr2
;
2313 int need_subpage
= 0;
2315 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2317 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2318 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2319 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2320 &p
->phys_offset
, orig_memory
,
2323 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2326 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
,
2328 p
->region_offset
= 0;
2330 p
->phys_offset
= phys_offset
;
2331 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2332 (phys_offset
& IO_MEM_ROMD
))
2333 phys_offset
+= TARGET_PAGE_SIZE
;
2336 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2337 p
->phys_offset
= phys_offset
;
2338 p
->region_offset
= region_offset
;
2339 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2340 (phys_offset
& IO_MEM_ROMD
)) {
2341 phys_offset
+= TARGET_PAGE_SIZE
;
2343 target_phys_addr_t start_addr2
, end_addr2
;
2344 int need_subpage
= 0;
2346 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2347 end_addr2
, need_subpage
);
2349 if (need_subpage
|| phys_offset
& IO_MEM_SUBWIDTH
) {
2350 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2351 &p
->phys_offset
, IO_MEM_UNASSIGNED
,
2352 addr
& TARGET_PAGE_MASK
);
2353 subpage_register(subpage
, start_addr2
, end_addr2
,
2354 phys_offset
, region_offset
);
2355 p
->region_offset
= 0;
2359 region_offset
+= TARGET_PAGE_SIZE
;
2362 /* since each CPU stores ram addresses in its TLB cache, we must
2363 reset the modified entries */
2365 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2370 /* XXX: temporary until new memory mapping API */
2371 ram_addr_t
cpu_get_physical_page_desc(target_phys_addr_t addr
)
2375 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2377 return IO_MEM_UNASSIGNED
;
2378 return p
->phys_offset
;
2381 void qemu_register_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2384 kvm_coalesce_mmio_region(addr
, size
);
2387 void qemu_unregister_coalesced_mmio(target_phys_addr_t addr
, ram_addr_t size
)
2390 kvm_uncoalesce_mmio_region(addr
, size
);
2393 /* XXX: better than nothing */
2394 ram_addr_t
qemu_ram_alloc(ram_addr_t size
)
2397 if ((phys_ram_alloc_offset
+ size
) > phys_ram_size
) {
2398 fprintf(stderr
, "Not enough memory (requested_size = %" PRIu64
", max memory = %" PRIu64
")\n",
2399 (uint64_t)size
, (uint64_t)phys_ram_size
);
2402 addr
= phys_ram_alloc_offset
;
2403 phys_ram_alloc_offset
= TARGET_PAGE_ALIGN(phys_ram_alloc_offset
+ size
);
2407 void qemu_ram_free(ram_addr_t addr
)
2411 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2413 #ifdef DEBUG_UNASSIGNED
2414 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2416 printf("unassigned_mem_readb Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2418 #if defined(TARGET_SPARC)
2419 do_unassigned_access(addr
, 0, 0, 0, 1);
2424 static uint32_t unassigned_mem_readw(void *opaque
, target_phys_addr_t addr
)
2426 #ifdef DEBUG_UNASSIGNED
2427 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2429 printf("unassigned_mem_readw Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2431 #if defined(TARGET_SPARC)
2432 do_unassigned_access(addr
, 0, 0, 0, 2);
2437 static uint32_t unassigned_mem_readl(void *opaque
, target_phys_addr_t addr
)
2439 #ifdef DEBUG_UNASSIGNED
2440 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2442 printf("unassigned_mem_readl Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2445 #if defined(TARGET_SPARC)
2446 do_unassigned_access(addr
, 0, 0, 0, 4);
2451 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2453 #ifdef DEBUG_UNASSIGNED
2454 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2457 printf("%s Unassigned mem read " TARGET_FMT_plx
" pc "TARGET_FMT_plx
"\n", __FUNCTION__
,addr
,cpu_single_env
->active_tc
.PC
);
2461 #if defined(TARGET_SPARC)
2462 do_unassigned_access(addr
, 1, 0, 0, 1);
2466 static void unassigned_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2468 #ifdef DEBUG_UNASSIGNED
2469 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2472 printf("%s Unassigned mem read " TARGET_FMT_plx
" pc "TARGET_FMT_plx
"\n", __FUNCTION__
,addr
,cpu_single_env
->active_tc
.PC
);
2476 #if defined(TARGET_SPARC)
2477 do_unassigned_access(addr
, 1, 0, 0, 2);
2481 static void unassigned_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2483 #ifdef DEBUG_UNASSIGNED
2484 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2487 printf("%s Unassigned mem read " TARGET_FMT_plx
" pc "TARGET_FMT_plx
"\n", __FUNCTION__
,addr
,cpu_single_env
->active_tc
.PC
);
2491 #if defined(TARGET_SPARC)
2492 do_unassigned_access(addr
, 1, 0, 0, 4);
2496 static CPUReadMemoryFunc
*unassigned_mem_read
[3] = {
2497 unassigned_mem_readb
,
2498 unassigned_mem_readw
,
2499 unassigned_mem_readl
,
2502 static CPUWriteMemoryFunc
*unassigned_mem_write
[3] = {
2503 unassigned_mem_writeb
,
2504 unassigned_mem_writew
,
2505 unassigned_mem_writel
,
2508 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t ram_addr
,
2512 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2513 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2514 #if !defined(CONFIG_USER_ONLY)
2515 tb_invalidate_phys_page_fast(ram_addr
, 1);
2516 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2519 stb_p(phys_ram_base
+ ram_addr
, val
);
2521 if (cpu_single_env
->kqemu_enabled
&&
2522 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2523 kqemu_modify_page(cpu_single_env
, ram_addr
);
2525 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2526 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2527 /* we remove the notdirty callback only if the code has been
2529 if (dirty_flags
== 0xff)
2530 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2533 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t ram_addr
,
2537 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2538 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2539 #if !defined(CONFIG_USER_ONLY)
2540 tb_invalidate_phys_page_fast(ram_addr
, 2);
2541 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2544 stw_p(phys_ram_base
+ ram_addr
, val
);
2546 if (cpu_single_env
->kqemu_enabled
&&
2547 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2548 kqemu_modify_page(cpu_single_env
, ram_addr
);
2550 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2551 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2552 /* we remove the notdirty callback only if the code has been
2554 if (dirty_flags
== 0xff)
2555 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2558 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t ram_addr
,
2562 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2563 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2564 #if !defined(CONFIG_USER_ONLY)
2565 tb_invalidate_phys_page_fast(ram_addr
, 4);
2566 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2569 stl_p(phys_ram_base
+ ram_addr
, val
);
2571 if (cpu_single_env
->kqemu_enabled
&&
2572 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2573 kqemu_modify_page(cpu_single_env
, ram_addr
);
2575 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2576 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2577 /* we remove the notdirty callback only if the code has been
2579 if (dirty_flags
== 0xff)
2580 tlb_set_dirty(cpu_single_env
, cpu_single_env
->mem_io_vaddr
);
2583 static CPUReadMemoryFunc
*error_mem_read
[3] = {
2584 NULL
, /* never used */
2585 NULL
, /* never used */
2586 NULL
, /* never used */
2589 static CPUWriteMemoryFunc
*notdirty_mem_write
[3] = {
2590 notdirty_mem_writeb
,
2591 notdirty_mem_writew
,
2592 notdirty_mem_writel
,
2595 /* Generate a debug exception if a watchpoint has been hit. */
2596 static void check_watchpoint(int offset
, int len_mask
, int flags
)
2598 CPUState
*env
= cpu_single_env
;
2599 target_ulong pc
, cs_base
;
2600 TranslationBlock
*tb
;
2605 if (env
->watchpoint_hit
) {
2606 /* We re-entered the check after replacing the TB. Now raise
2607 * the debug interrupt so that is will trigger after the
2608 * current instruction. */
2609 cpu_interrupt(env
, CPU_INTERRUPT_DEBUG
);
2612 vaddr
= (env
->mem_io_vaddr
& TARGET_PAGE_MASK
) + offset
;
2613 TAILQ_FOREACH(wp
, &env
->watchpoints
, entry
) {
2614 if ((vaddr
== (wp
->vaddr
& len_mask
) ||
2615 (vaddr
& wp
->len_mask
) == wp
->vaddr
) && (wp
->flags
& flags
)) {
2616 wp
->flags
|= BP_WATCHPOINT_HIT
;
2617 if (!env
->watchpoint_hit
) {
2618 env
->watchpoint_hit
= wp
;
2619 tb
= tb_find_pc(env
->mem_io_pc
);
2621 cpu_abort(env
, "check_watchpoint: could not find TB for "
2622 "pc=%p", (void *)env
->mem_io_pc
);
2624 cpu_restore_state(tb
, env
, env
->mem_io_pc
, NULL
);
2625 tb_phys_invalidate(tb
, -1);
2626 if (wp
->flags
& BP_STOP_BEFORE_ACCESS
) {
2627 env
->exception_index
= EXCP_DEBUG
;
2629 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &cpu_flags
);
2630 tb_gen_code(env
, pc
, cs_base
, cpu_flags
, 1);
2632 cpu_resume_from_signal(env
, NULL
);
2635 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
2640 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2641 so these check for a hit then pass through to the normal out-of-line
2643 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
2645 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_READ
);
2646 return ldub_phys(addr
);
2649 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
2651 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_READ
);
2652 return lduw_phys(addr
);
2655 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
2657 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_READ
);
2658 return ldl_phys(addr
);
2661 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
2664 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x0, BP_MEM_WRITE
);
2665 stb_phys(addr
, val
);
2668 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
2671 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x1, BP_MEM_WRITE
);
2672 stw_phys(addr
, val
);
2675 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
2678 check_watchpoint(addr
& ~TARGET_PAGE_MASK
, ~0x3, BP_MEM_WRITE
);
2679 stl_phys(addr
, val
);
2682 static CPUReadMemoryFunc
*watch_mem_read
[3] = {
2688 static CPUWriteMemoryFunc
*watch_mem_write
[3] = {
2694 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
2700 idx
= SUBPAGE_IDX(addr
);
2701 #if defined(DEBUG_SUBPAGE)
2702 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
2703 mmio
, len
, addr
, idx
);
2705 ret
= (**mmio
->mem_read
[idx
][len
])(mmio
->opaque
[idx
][0][len
],
2706 addr
+ mmio
->region_offset
[idx
][0][len
]);
2711 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
2712 uint32_t value
, unsigned int len
)
2716 idx
= SUBPAGE_IDX(addr
);
2717 #if defined(DEBUG_SUBPAGE)
2718 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
2719 mmio
, len
, addr
, idx
, value
);
2721 (**mmio
->mem_write
[idx
][len
])(mmio
->opaque
[idx
][1][len
],
2722 addr
+ mmio
->region_offset
[idx
][1][len
],
2726 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
2728 #if defined(DEBUG_SUBPAGE)
2729 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2732 return subpage_readlen(opaque
, addr
, 0);
2735 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
2738 #if defined(DEBUG_SUBPAGE)
2739 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2741 subpage_writelen(opaque
, addr
, value
, 0);
2744 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
2746 #if defined(DEBUG_SUBPAGE)
2747 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2750 return subpage_readlen(opaque
, addr
, 1);
2753 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
2756 #if defined(DEBUG_SUBPAGE)
2757 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2759 subpage_writelen(opaque
, addr
, value
, 1);
2762 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
2764 #if defined(DEBUG_SUBPAGE)
2765 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2768 return subpage_readlen(opaque
, addr
, 2);
2771 static void subpage_writel (void *opaque
,
2772 target_phys_addr_t addr
, uint32_t value
)
2774 #if defined(DEBUG_SUBPAGE)
2775 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2777 subpage_writelen(opaque
, addr
, value
, 2);
2780 static CPUReadMemoryFunc
*subpage_read
[] = {
2786 static CPUWriteMemoryFunc
*subpage_write
[] = {
2792 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2793 ram_addr_t memory
, ram_addr_t region_offset
)
2798 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2800 idx
= SUBPAGE_IDX(start
);
2801 eidx
= SUBPAGE_IDX(end
);
2802 #if defined(DEBUG_SUBPAGE)
2803 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__
,
2804 mmio
, start
, end
, idx
, eidx
, memory
);
2806 memory
>>= IO_MEM_SHIFT
;
2807 for (; idx
<= eidx
; idx
++) {
2808 for (i
= 0; i
< 4; i
++) {
2809 printf("memory %llx \n",memory
);
2811 if (io_mem_read
[memory
][i
]) {
2812 mmio
->mem_read
[idx
][i
] = &io_mem_read
[memory
][i
];
2813 mmio
->opaque
[idx
][0][i
] = io_mem_opaque
[memory
];
2814 mmio
->region_offset
[idx
][0][i
] = region_offset
;
2816 if (io_mem_write
[memory
][i
]) {
2817 mmio
->mem_write
[idx
][i
] = &io_mem_write
[memory
][i
];
2818 mmio
->opaque
[idx
][1][i
] = io_mem_opaque
[memory
];
2819 mmio
->region_offset
[idx
][1][i
] = region_offset
;
2827 static void *subpage_init (target_phys_addr_t base
, ram_addr_t
*phys
,
2828 ram_addr_t orig_memory
, ram_addr_t region_offset
)
2833 mmio
= qemu_mallocz(sizeof(subpage_t
));
2836 subpage_memory
= cpu_register_io_memory(0, subpage_read
, subpage_write
, mmio
);
2837 #if defined(DEBUG_SUBPAGE)
2838 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
2839 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
2841 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
2842 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
,
2848 static int get_free_io_mem_idx(void)
2852 for (i
= 0; i
<IO_MEM_NB_ENTRIES
; i
++)
2853 if (!io_mem_used
[i
]) {
2861 static void io_mem_init(void)
2865 cpu_register_io_memory(IO_MEM_ROM
>> IO_MEM_SHIFT
, error_mem_read
, unassigned_mem_write
, NULL
);
2866 cpu_register_io_memory(IO_MEM_UNASSIGNED
>> IO_MEM_SHIFT
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
2867 cpu_register_io_memory(IO_MEM_NOTDIRTY
>> IO_MEM_SHIFT
, error_mem_read
, notdirty_mem_write
, NULL
);
2871 io_mem_watch
= cpu_register_io_memory(0, watch_mem_read
,
2872 watch_mem_write
, NULL
);
2873 /* alloc dirty bits array */
2874 phys_ram_dirty
= qemu_vmalloc(phys_ram_size
>> TARGET_PAGE_BITS
);
2875 memset(phys_ram_dirty
, 0xff, phys_ram_size
>> TARGET_PAGE_BITS
);
2878 /* mem_read and mem_write are arrays of functions containing the
2879 function to access byte (index 0), word (index 1) and dword (index
2880 2). Functions can be omitted with a NULL function pointer. The
2881 registered functions may be modified dynamically later.
2882 If io_index is non zero, the corresponding io zone is
2883 modified. If it is zero, a new io zone is allocated. The return
2884 value can be used with cpu_register_physical_memory(). (-1) is
2885 returned if error. */
2886 int cpu_register_io_memory(int io_index
,
2887 CPUReadMemoryFunc
**mem_read
,
2888 CPUWriteMemoryFunc
**mem_write
,
2891 int i
, subwidth
= 0;
2893 if (io_index
<= 0) {
2894 io_index
= get_free_io_mem_idx();
2898 if (io_index
>= IO_MEM_NB_ENTRIES
)
2902 for(i
= 0;i
< 3; i
++) {
2903 if (!mem_read
[i
] || !mem_write
[i
])
2904 subwidth
= IO_MEM_SUBWIDTH
;
2905 io_mem_read
[io_index
][i
] = mem_read
[i
];
2906 io_mem_write
[io_index
][i
] = mem_write
[i
];
2908 io_mem_opaque
[io_index
] = opaque
;
2909 return (io_index
<< IO_MEM_SHIFT
) | subwidth
;
2912 void cpu_unregister_io_memory(int io_table_address
)
2915 int io_index
= io_table_address
>> IO_MEM_SHIFT
;
2917 for (i
=0;i
< 3; i
++) {
2918 io_mem_read
[io_index
][i
] = unassigned_mem_read
[i
];
2919 io_mem_write
[io_index
][i
] = unassigned_mem_write
[i
];
2921 io_mem_opaque
[io_index
] = NULL
;
2922 io_mem_used
[io_index
] = 0;
2925 CPUWriteMemoryFunc
**cpu_get_io_memory_write(int io_index
)
2927 return io_mem_write
[io_index
>> IO_MEM_SHIFT
];
2930 CPUReadMemoryFunc
**cpu_get_io_memory_read(int io_index
)
2932 return io_mem_read
[io_index
>> IO_MEM_SHIFT
];
2935 #endif /* !defined(CONFIG_USER_ONLY) */
2937 /* physical memory access (slow version, mainly for debug) */
2938 #if defined(CONFIG_USER_ONLY)
2939 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2940 int len
, int is_write
)
2947 page
= addr
& TARGET_PAGE_MASK
;
2948 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2951 flags
= page_get_flags(page
);
2952 if (!(flags
& PAGE_VALID
))
2955 if (!(flags
& PAGE_WRITE
))
2957 /* XXX: this code should not depend on lock_user */
2958 if (!(p
= lock_user(VERIFY_WRITE
, addr
, l
, 0)))
2959 /* FIXME - should this return an error rather than just fail? */
2962 unlock_user(p
, addr
, l
);
2964 if (!(flags
& PAGE_READ
))
2966 /* XXX: this code should not depend on lock_user */
2967 if (!(p
= lock_user(VERIFY_READ
, addr
, l
, 1)))
2968 /* FIXME - should this return an error rather than just fail? */
2971 unlock_user(p
, addr
, 0);
2980 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2981 int len
, int is_write
)
2986 target_phys_addr_t page
;
2991 page
= addr
& TARGET_PAGE_MASK
;
2992 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2995 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2997 pd
= IO_MEM_UNASSIGNED
;
2999 pd
= p
->phys_offset
;
3003 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3004 target_phys_addr_t addr1
= addr
;
3005 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3007 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3008 /* XXX: could force cpu_single_env to NULL to avoid
3010 if (l
>= 4 && ((addr1
& 3) == 0)) {
3011 /* 32 bit write access */
3013 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr1
, val
);
3015 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3016 /* 16 bit write access */
3018 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr1
, val
);
3021 /* 8 bit write access */
3023 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr1
, val
);
3027 unsigned long addr1
;
3028 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3030 ptr
= phys_ram_base
+ addr1
;
3031 memcpy(ptr
, buf
, l
);
3032 if (!cpu_physical_memory_is_dirty(addr1
)) {
3033 /* invalidate code */
3034 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3036 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3037 (0xff & ~CODE_DIRTY_FLAG
);
3041 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3042 !(pd
& IO_MEM_ROMD
)) {
3043 target_phys_addr_t addr1
= addr
;
3045 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3047 addr1
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3048 if (l
>= 4 && ((addr1
& 3) == 0)) {
3049 /* 32 bit read access */
3050 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr1
);
3053 } else if (l
>= 2 && ((addr1
& 1) == 0)) {
3054 /* 16 bit read access */
3055 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr1
);
3059 /* 8 bit read access */
3060 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr1
);
3066 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3067 (addr
& ~TARGET_PAGE_MASK
);
3068 memcpy(buf
, ptr
, l
);
3077 /* used for ROM loading : can write in RAM and ROM */
3078 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
3079 const uint8_t *buf
, int len
)
3083 target_phys_addr_t page
;
3088 page
= addr
& TARGET_PAGE_MASK
;
3089 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3092 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3094 pd
= IO_MEM_UNASSIGNED
;
3096 pd
= p
->phys_offset
;
3099 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
3100 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
3101 !(pd
& IO_MEM_ROMD
)) {
3104 unsigned long addr1
;
3105 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3107 ptr
= phys_ram_base
+ addr1
;
3108 memcpy(ptr
, buf
, l
);
3118 target_phys_addr_t addr
;
3119 target_phys_addr_t len
;
3122 static BounceBuffer bounce
;
3124 typedef struct MapClient
{
3126 void (*callback
)(void *opaque
);
3127 LIST_ENTRY(MapClient
) link
;
3130 static LIST_HEAD(map_client_list
, MapClient
) map_client_list
3131 = LIST_HEAD_INITIALIZER(map_client_list
);
3133 void *cpu_register_map_client(void *opaque
, void (*callback
)(void *opaque
))
3135 MapClient
*client
= qemu_malloc(sizeof(*client
));
3137 client
->opaque
= opaque
;
3138 client
->callback
= callback
;
3139 LIST_INSERT_HEAD(&map_client_list
, client
, link
);
3143 void cpu_unregister_map_client(void *_client
)
3145 MapClient
*client
= (MapClient
*)_client
;
3147 LIST_REMOVE(client
, link
);
3150 static void cpu_notify_map_clients(void)
3154 while (!LIST_EMPTY(&map_client_list
)) {
3155 client
= LIST_FIRST(&map_client_list
);
3156 client
->callback(client
->opaque
);
3157 LIST_REMOVE(client
, link
);
3161 /* Map a physical memory region into a host virtual address.
3162 * May map a subset of the requested range, given by and returned in *plen.
3163 * May return NULL if resources needed to perform the mapping are exhausted.
3164 * Use only for reads OR writes - not for read-modify-write operations.
3165 * Use cpu_register_map_client() to know when retrying the map operation is
3166 * likely to succeed.
3168 void *cpu_physical_memory_map(target_phys_addr_t addr
,
3169 target_phys_addr_t
*plen
,
3172 target_phys_addr_t len
= *plen
;
3173 target_phys_addr_t done
= 0;
3175 uint8_t *ret
= NULL
;
3177 target_phys_addr_t page
;
3180 unsigned long addr1
;
3183 page
= addr
& TARGET_PAGE_MASK
;
3184 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3187 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
3189 pd
= IO_MEM_UNASSIGNED
;
3191 pd
= p
->phys_offset
;
3194 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3195 if (done
|| bounce
.buffer
) {
3198 bounce
.buffer
= qemu_memalign(TARGET_PAGE_SIZE
, TARGET_PAGE_SIZE
);
3202 cpu_physical_memory_rw(addr
, bounce
.buffer
, l
, 0);
3204 ptr
= bounce
.buffer
;
3206 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3207 ptr
= phys_ram_base
+ addr1
;
3211 } else if (ret
+ done
!= ptr
) {
3223 /* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3224 * Will also mark the memory as dirty if is_write == 1. access_len gives
3225 * the amount of memory that was actually read or written by the caller.
3227 void cpu_physical_memory_unmap(void *buffer
, target_phys_addr_t len
,
3228 int is_write
, target_phys_addr_t access_len
)
3230 if (buffer
!= bounce
.buffer
) {
3232 unsigned long addr1
= (uint8_t *)buffer
- phys_ram_base
;
3233 while (access_len
) {
3235 l
= TARGET_PAGE_SIZE
;
3238 if (!cpu_physical_memory_is_dirty(addr1
)) {
3239 /* invalidate code */
3240 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
3242 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3243 (0xff & ~CODE_DIRTY_FLAG
);
3252 cpu_physical_memory_write(bounce
.addr
, bounce
.buffer
, access_len
);
3254 qemu_free(bounce
.buffer
);
3255 bounce
.buffer
= NULL
;
3256 cpu_notify_map_clients();
3259 /* warning: addr must be aligned */
3260 uint32_t ldl_phys(target_phys_addr_t addr
)
3268 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3270 pd
= IO_MEM_UNASSIGNED
;
3272 pd
= p
->phys_offset
;
3275 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3276 !(pd
& IO_MEM_ROMD
)) {
3278 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3280 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3281 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3284 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3285 (addr
& ~TARGET_PAGE_MASK
);
3291 /* warning: addr must be aligned */
3292 uint64_t ldq_phys(target_phys_addr_t addr
)
3300 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3302 pd
= IO_MEM_UNASSIGNED
;
3304 pd
= p
->phys_offset
;
3307 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
3308 !(pd
& IO_MEM_ROMD
)) {
3310 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3312 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3313 #ifdef TARGET_WORDS_BIGENDIAN
3314 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
3315 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
3317 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
3318 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
3322 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3323 (addr
& ~TARGET_PAGE_MASK
);
3330 uint32_t ldub_phys(target_phys_addr_t addr
)
3333 cpu_physical_memory_read(addr
, &val
, 1);
3338 uint32_t lduw_phys(target_phys_addr_t addr
)
3341 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
3342 return tswap16(val
);
3345 /* warning: addr must be aligned. The ram page is not masked as dirty
3346 and the code inside is not invalidated. It is useful if the dirty
3347 bits are used to track modified PTEs */
3348 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
3355 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3357 pd
= IO_MEM_UNASSIGNED
;
3359 pd
= p
->phys_offset
;
3362 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3363 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3365 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3366 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3368 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3369 ptr
= phys_ram_base
+ addr1
;
3372 if (unlikely(in_migration
)) {
3373 if (!cpu_physical_memory_is_dirty(addr1
)) {
3374 /* invalidate code */
3375 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3377 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3378 (0xff & ~CODE_DIRTY_FLAG
);
3384 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
3391 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3393 pd
= IO_MEM_UNASSIGNED
;
3395 pd
= p
->phys_offset
;
3398 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3399 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3401 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3402 #ifdef TARGET_WORDS_BIGENDIAN
3403 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
3404 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
3406 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3407 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
3410 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
3411 (addr
& ~TARGET_PAGE_MASK
);
3416 /* warning: addr must be aligned */
3417 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
3424 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
3426 pd
= IO_MEM_UNASSIGNED
;
3428 pd
= p
->phys_offset
;
3431 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
3432 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
3434 addr
= (addr
& ~TARGET_PAGE_MASK
) + p
->region_offset
;
3435 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
3437 unsigned long addr1
;
3438 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
3440 ptr
= phys_ram_base
+ addr1
;
3442 if (!cpu_physical_memory_is_dirty(addr1
)) {
3443 /* invalidate code */
3444 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
3446 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
3447 (0xff & ~CODE_DIRTY_FLAG
);
3453 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
3456 cpu_physical_memory_write(addr
, &v
, 1);
3460 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
3462 uint16_t v
= tswap16(val
);
3463 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
3467 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
3470 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
3475 /* virtual memory access for debug */
3476 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
3477 uint8_t *buf
, int len
, int is_write
)
3480 target_phys_addr_t phys_addr
;
3484 page
= addr
& TARGET_PAGE_MASK
;
3485 phys_addr
= cpu_get_phys_page_debug(env
, page
);
3486 /* if no physical page mapped, return an error */
3487 if (phys_addr
== -1)
3489 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
3492 cpu_physical_memory_rw(phys_addr
+ (addr
& ~TARGET_PAGE_MASK
),
3501 /* in deterministic execution mode, instructions doing device I/Os
3502 must be at the end of the TB */
3503 void cpu_io_recompile(CPUState
*env
, void *retaddr
)
3505 TranslationBlock
*tb
;
3507 target_ulong pc
, cs_base
;
3510 tb
= tb_find_pc((unsigned long)retaddr
);
3512 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
3515 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
3516 cpu_restore_state(tb
, env
, (unsigned long)retaddr
, NULL
);
3517 /* Calculate how many instructions had been executed before the fault
3519 n
= n
- env
->icount_decr
.u16
.low
;
3520 /* Generate a new TB ending on the I/O insn. */
3522 /* On MIPS and SH, delay slot instructions can only be restarted if
3523 they were already the first instruction in the TB. If this is not
3524 the first instruction in a TB then re-execute the preceding
3526 #if defined(TARGET_MIPS)
3527 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
3528 env
->active_tc
.PC
-= 4;
3529 env
->icount_decr
.u16
.low
++;
3530 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
3532 #elif defined(TARGET_SH4)
3533 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
3536 env
->icount_decr
.u16
.low
++;
3537 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
3540 /* This should never happen. */
3541 if (n
> CF_COUNT_MASK
)
3542 cpu_abort(env
, "TB too big during recompile");
3544 cflags
= n
| CF_LAST_IO
;
3546 cs_base
= tb
->cs_base
;
3548 tb_phys_invalidate(tb
, -1);
3549 /* FIXME: In theory this could raise an exception. In practice
3550 we have already translated the block once so it's probably ok. */
3551 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
3552 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
3553 the first in the TB) then we end up generating a whole new TB and
3554 repeating the fault, which is horribly inefficient.
3555 Better would be to execute just this insn uncached, or generate a
3557 cpu_resume_from_signal(env
, NULL
);
3560 void dump_exec_info(FILE *f
,
3561 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
3563 int i
, target_code_size
, max_target_code_size
;
3564 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
3565 TranslationBlock
*tb
;
3567 target_code_size
= 0;
3568 max_target_code_size
= 0;
3570 direct_jmp_count
= 0;
3571 direct_jmp2_count
= 0;
3572 for(i
= 0; i
< nb_tbs
; i
++) {
3574 target_code_size
+= tb
->size
;
3575 if (tb
->size
> max_target_code_size
)
3576 max_target_code_size
= tb
->size
;
3577 if (tb
->page_addr
[1] != -1)
3579 if (tb
->tb_next_offset
[0] != 0xffff) {
3581 if (tb
->tb_next_offset
[1] != 0xffff) {
3582 direct_jmp2_count
++;
3586 /* XXX: avoid using doubles ? */
3587 cpu_fprintf(f
, "Translation buffer state:\n");
3588 cpu_fprintf(f
, "gen code size %ld/%ld\n",
3589 code_gen_ptr
- code_gen_buffer
, code_gen_buffer_max_size
);
3590 cpu_fprintf(f
, "TB count %d/%d\n",
3591 nb_tbs
, code_gen_max_blocks
);
3592 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
3593 nb_tbs
? target_code_size
/ nb_tbs
: 0,
3594 max_target_code_size
);
3595 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3596 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
3597 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
3598 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
3600 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
3601 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3603 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
3605 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
3606 cpu_fprintf(f
, "\nStatistics:\n");
3607 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
3608 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
3609 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
3610 tcg_dump_info(f
, cpu_fprintf
);
3613 #if !defined(CONFIG_USER_ONLY)
3615 #define MMUSUFFIX _cmmu
3616 #define GETPC() NULL
3617 #define env cpu_single_env
3618 #define SOFTMMU_CODE_ACCESS
3621 #include "softmmu_template.h"
3624 #include "softmmu_template.h"
3627 #include "softmmu_template.h"
3630 #include "softmmu_template.h"