2 * virtual page mapping and translated block handling
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 #define WIN32_LEAN_AND_MEAN
25 #include <sys/types.h>
42 #if defined(CONFIG_USER_ONLY)
46 //#define DEBUG_TB_INVALIDATE
49 //#define DEBUG_UNASSIGNED
51 /* make various TB consistency checks */
52 //#define DEBUG_TB_CHECK
53 //#define DEBUG_TLB_CHECK
55 //#define DEBUG_IOPORT
56 //#define DEBUG_SUBPAGE
58 #if !defined(CONFIG_USER_ONLY)
59 /* TB consistency checks only implemented for usermode emulation. */
63 /* threshold to flush the translated code buffer */
64 #define CODE_GEN_BUFFER_MAX_SIZE (CODE_GEN_BUFFER_SIZE - code_gen_max_block_size())
66 #define SMC_BITMAP_USE_THRESHOLD 10
68 #define MMAP_AREA_START 0x00000000
69 #define MMAP_AREA_END 0xa8000000
71 #if defined(TARGET_SPARC64)
72 #define TARGET_PHYS_ADDR_SPACE_BITS 41
73 #elif defined(TARGET_SPARC)
74 #define TARGET_PHYS_ADDR_SPACE_BITS 36
75 #elif defined(TARGET_ALPHA)
76 #define TARGET_PHYS_ADDR_SPACE_BITS 42
77 #define TARGET_VIRT_ADDR_SPACE_BITS 42
78 #elif defined(TARGET_PPC64)
79 #define TARGET_PHYS_ADDR_SPACE_BITS 42
81 /* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
82 #define TARGET_PHYS_ADDR_SPACE_BITS 32
84 #define TARGET_PHYS_ADDR_SPACE_BITS 42
85 #elif defined(TARGET_IA64)
86 #define TARGET_PHYS_ADDR_SPACE_BITS 36
88 #define TARGET_PHYS_ADDR_SPACE_BITS 32
92 extern int kvm_allowed
;
93 extern kvm_context_t kvm_context
;
96 TranslationBlock tbs
[CODE_GEN_MAX_BLOCKS
];
97 TranslationBlock
*tb_phys_hash
[CODE_GEN_PHYS_HASH_SIZE
];
99 /* any access to the tbs or the page table must use this lock */
100 spinlock_t tb_lock
= SPIN_LOCK_UNLOCKED
;
102 uint8_t code_gen_buffer
[CODE_GEN_BUFFER_SIZE
] __attribute__((aligned (32)));
103 uint8_t *code_gen_ptr
;
105 ram_addr_t phys_ram_size
;
107 uint8_t *phys_ram_base
;
108 uint8_t *phys_ram_dirty
;
110 static int in_migration
;
111 static ram_addr_t phys_ram_alloc_offset
= 0;
114 /* current CPU in the current thread. It is only valid inside
116 CPUState
*cpu_single_env
;
118 typedef struct PageDesc
{
119 /* list of TBs intersecting this ram page */
120 TranslationBlock
*first_tb
;
121 /* in order to optimize self modifying code, we count the number
122 of lookups we do to a given page to use a bitmap */
123 unsigned int code_write_count
;
124 uint8_t *code_bitmap
;
125 #if defined(CONFIG_USER_ONLY)
130 typedef struct PhysPageDesc
{
131 /* offset in host memory of the page + io_index in the low 12 bits */
132 ram_addr_t phys_offset
;
136 #if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
137 /* XXX: this is a temporary hack for alpha target.
138 * In the future, this is to be replaced by a multi-level table
139 * to actually be able to handle the complete 64 bits address space.
141 #define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
143 #define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
146 #define L1_SIZE (1 << L1_BITS)
147 #define L2_SIZE (1 << L2_BITS)
149 static void io_mem_init(void);
151 unsigned long qemu_real_host_page_size
;
152 unsigned long qemu_host_page_bits
;
153 unsigned long qemu_host_page_size
;
154 unsigned long qemu_host_page_mask
;
156 /* XXX: for system emulation, it could just be an array */
157 static PageDesc
*l1_map
[L1_SIZE
];
158 PhysPageDesc
**l1_phys_map
;
160 /* io memory support */
161 CPUWriteMemoryFunc
*io_mem_write
[IO_MEM_NB_ENTRIES
][4];
162 CPUReadMemoryFunc
*io_mem_read
[IO_MEM_NB_ENTRIES
][4];
163 void *io_mem_opaque
[IO_MEM_NB_ENTRIES
];
164 static int io_mem_nb
;
165 #if defined(CONFIG_SOFTMMU)
166 static int io_mem_watch
;
170 char *logfilename
= "/tmp/qemu.log";
173 static int log_append
= 0;
176 static int tlb_flush_count
;
177 static int tb_flush_count
;
178 static int tb_phys_invalidate_count
;
180 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
181 typedef struct subpage_t
{
182 target_phys_addr_t base
;
183 CPUReadMemoryFunc
**mem_read
[TARGET_PAGE_SIZE
];
184 CPUWriteMemoryFunc
**mem_write
[TARGET_PAGE_SIZE
];
185 void *opaque
[TARGET_PAGE_SIZE
];
188 static void page_init(void)
190 /* NOTE: we can always suppose that qemu_host_page_size >=
194 SYSTEM_INFO system_info
;
197 GetSystemInfo(&system_info
);
198 qemu_real_host_page_size
= system_info
.dwPageSize
;
200 VirtualProtect(code_gen_buffer
, sizeof(code_gen_buffer
),
201 PAGE_EXECUTE_READWRITE
, &old_protect
);
204 qemu_real_host_page_size
= getpagesize();
206 unsigned long start
, end
;
208 start
= (unsigned long)code_gen_buffer
;
209 start
&= ~(qemu_real_host_page_size
- 1);
211 end
= (unsigned long)code_gen_buffer
+ sizeof(code_gen_buffer
);
212 end
+= qemu_real_host_page_size
- 1;
213 end
&= ~(qemu_real_host_page_size
- 1);
215 mprotect((void *)start
, end
- start
,
216 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
220 if (qemu_host_page_size
== 0)
221 qemu_host_page_size
= qemu_real_host_page_size
;
222 if (qemu_host_page_size
< TARGET_PAGE_SIZE
)
223 qemu_host_page_size
= TARGET_PAGE_SIZE
;
224 qemu_host_page_bits
= 0;
225 while ((1 << qemu_host_page_bits
) < qemu_host_page_size
)
226 qemu_host_page_bits
++;
227 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
228 l1_phys_map
= qemu_vmalloc(L1_SIZE
* sizeof(void *));
229 memset(l1_phys_map
, 0, L1_SIZE
* sizeof(void *));
231 #if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
233 long long startaddr
, endaddr
;
237 f
= fopen("/proc/self/maps", "r");
240 n
= fscanf (f
, "%llx-%llx %*[^\n]\n", &startaddr
, &endaddr
);
242 page_set_flags(TARGET_PAGE_ALIGN(startaddr
),
243 TARGET_PAGE_ALIGN(endaddr
),
253 static inline PageDesc
*page_find_alloc(unsigned int index
)
257 lp
= &l1_map
[index
>> L2_BITS
];
260 /* allocate if not found */
261 p
= qemu_malloc(sizeof(PageDesc
) * L2_SIZE
);
262 memset(p
, 0, sizeof(PageDesc
) * L2_SIZE
);
265 return p
+ (index
& (L2_SIZE
- 1));
268 static inline PageDesc
*page_find(unsigned int index
)
272 p
= l1_map
[index
>> L2_BITS
];
275 return p
+ (index
& (L2_SIZE
- 1));
278 static PhysPageDesc
*phys_page_find_alloc(target_phys_addr_t index
, int alloc
)
283 p
= (void **)l1_phys_map
;
284 #if TARGET_PHYS_ADDR_SPACE_BITS > 32
286 #if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
287 #error unsupported TARGET_PHYS_ADDR_SPACE_BITS
289 lp
= p
+ ((index
>> (L1_BITS
+ L2_BITS
)) & (L1_SIZE
- 1));
292 /* allocate if not found */
295 p
= qemu_vmalloc(sizeof(void *) * L1_SIZE
);
296 memset(p
, 0, sizeof(void *) * L1_SIZE
);
300 lp
= p
+ ((index
>> L2_BITS
) & (L1_SIZE
- 1));
304 /* allocate if not found */
307 pd
= qemu_vmalloc(sizeof(PhysPageDesc
) * L2_SIZE
);
309 for (i
= 0; i
< L2_SIZE
; i
++)
310 pd
[i
].phys_offset
= IO_MEM_UNASSIGNED
;
312 return ((PhysPageDesc
*)pd
) + (index
& (L2_SIZE
- 1));
315 static inline PhysPageDesc
*phys_page_find(target_phys_addr_t index
)
317 return phys_page_find_alloc(index
, 0);
320 #if !defined(CONFIG_USER_ONLY)
321 static void tlb_protect_code(ram_addr_t ram_addr
);
322 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
326 void cpu_exec_init(CPUState
*env
)
332 code_gen_ptr
= code_gen_buffer
;
336 env
->next_cpu
= NULL
;
339 while (*penv
!= NULL
) {
340 penv
= (CPUState
**)&(*penv
)->next_cpu
;
343 env
->cpu_index
= cpu_index
;
344 env
->nb_watchpoints
= 0;
348 static inline void invalidate_page_bitmap(PageDesc
*p
)
350 if (p
->code_bitmap
) {
351 qemu_free(p
->code_bitmap
);
352 p
->code_bitmap
= NULL
;
354 p
->code_write_count
= 0;
357 /* set to NULL all the 'first_tb' fields in all PageDescs */
358 static void page_flush_tb(void)
363 for(i
= 0; i
< L1_SIZE
; i
++) {
366 for(j
= 0; j
< L2_SIZE
; j
++) {
368 invalidate_page_bitmap(p
);
375 /* flush all the translation blocks */
376 /* XXX: tb_flush is currently not thread safe */
377 void tb_flush(CPUState
*env1
)
380 #if defined(DEBUG_FLUSH)
381 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
382 (unsigned long)(code_gen_ptr
- code_gen_buffer
),
384 ((unsigned long)(code_gen_ptr
- code_gen_buffer
)) / nb_tbs
: 0);
388 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
389 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
392 memset (tb_phys_hash
, 0, CODE_GEN_PHYS_HASH_SIZE
* sizeof (void *));
395 code_gen_ptr
= code_gen_buffer
;
396 /* XXX: flush processor icache at this point if cache flush is
401 #ifdef DEBUG_TB_CHECK
403 static void tb_invalidate_check(target_ulong address
)
405 TranslationBlock
*tb
;
407 address
&= TARGET_PAGE_MASK
;
408 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
409 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
410 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
411 address
>= tb
->pc
+ tb
->size
)) {
412 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
413 address
, (long)tb
->pc
, tb
->size
);
419 /* verify that all the pages have correct rights for code */
420 static void tb_page_check(void)
422 TranslationBlock
*tb
;
423 int i
, flags1
, flags2
;
425 for(i
= 0;i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
426 for(tb
= tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
427 flags1
= page_get_flags(tb
->pc
);
428 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
429 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
430 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
431 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
437 void tb_jmp_check(TranslationBlock
*tb
)
439 TranslationBlock
*tb1
;
442 /* suppress any remaining jumps to this TB */
446 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
449 tb1
= tb1
->jmp_next
[n1
];
451 /* check end of list */
453 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb
);
459 /* invalidate one TB */
460 static inline void tb_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
,
463 TranslationBlock
*tb1
;
467 *ptb
= *(TranslationBlock
**)((char *)tb1
+ next_offset
);
470 ptb
= (TranslationBlock
**)((char *)tb1
+ next_offset
);
474 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
476 TranslationBlock
*tb1
;
482 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
484 *ptb
= tb1
->page_next
[n1
];
487 ptb
= &tb1
->page_next
[n1
];
491 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
493 TranslationBlock
*tb1
, **ptb
;
496 ptb
= &tb
->jmp_next
[n
];
499 /* find tb(n) in circular list */
503 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
504 if (n1
== n
&& tb1
== tb
)
507 ptb
= &tb1
->jmp_first
;
509 ptb
= &tb1
->jmp_next
[n1
];
512 /* now we can suppress tb(n) from the list */
513 *ptb
= tb
->jmp_next
[n
];
515 tb
->jmp_next
[n
] = NULL
;
519 /* reset the jump entry 'n' of a TB so that it is not chained to
521 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
523 tb_set_jmp_target(tb
, n
, (unsigned long)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
526 static inline void tb_phys_invalidate(TranslationBlock
*tb
, unsigned int page_addr
)
531 target_ulong phys_pc
;
532 TranslationBlock
*tb1
, *tb2
;
534 /* remove the TB from the hash list */
535 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
536 h
= tb_phys_hash_func(phys_pc
);
537 tb_remove(&tb_phys_hash
[h
], tb
,
538 offsetof(TranslationBlock
, phys_hash_next
));
540 /* remove the TB from the page list */
541 if (tb
->page_addr
[0] != page_addr
) {
542 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
543 tb_page_remove(&p
->first_tb
, tb
);
544 invalidate_page_bitmap(p
);
546 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
547 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
548 tb_page_remove(&p
->first_tb
, tb
);
549 invalidate_page_bitmap(p
);
552 tb_invalidated_flag
= 1;
554 /* remove the TB from the hash list */
555 h
= tb_jmp_cache_hash_func(tb
->pc
);
556 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
557 if (env
->tb_jmp_cache
[h
] == tb
)
558 env
->tb_jmp_cache
[h
] = NULL
;
561 /* suppress this TB from the two jump lists */
562 tb_jmp_remove(tb
, 0);
563 tb_jmp_remove(tb
, 1);
565 /* suppress any remaining jumps to this TB */
571 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
572 tb2
= tb1
->jmp_next
[n1
];
573 tb_reset_jump(tb1
, n1
);
574 tb1
->jmp_next
[n1
] = NULL
;
577 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2); /* fail safe */
579 tb_phys_invalidate_count
++;
582 static inline void set_bits(uint8_t *tab
, int start
, int len
)
588 mask
= 0xff << (start
& 7);
589 if ((start
& ~7) == (end
& ~7)) {
591 mask
&= ~(0xff << (end
& 7));
596 start
= (start
+ 8) & ~7;
598 while (start
< end1
) {
603 mask
= ~(0xff << (end
& 7));
609 static void build_page_bitmap(PageDesc
*p
)
611 int n
, tb_start
, tb_end
;
612 TranslationBlock
*tb
;
614 p
->code_bitmap
= qemu_malloc(TARGET_PAGE_SIZE
/ 8);
617 memset(p
->code_bitmap
, 0, TARGET_PAGE_SIZE
/ 8);
622 tb
= (TranslationBlock
*)((long)tb
& ~3);
623 /* NOTE: this is subtle as a TB may span two physical pages */
625 /* NOTE: tb_end may be after the end of the page, but
626 it is not a problem */
627 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
628 tb_end
= tb_start
+ tb
->size
;
629 if (tb_end
> TARGET_PAGE_SIZE
)
630 tb_end
= TARGET_PAGE_SIZE
;
633 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
635 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
636 tb
= tb
->page_next
[n
];
640 #ifdef TARGET_HAS_PRECISE_SMC
642 static void tb_gen_code(CPUState
*env
,
643 target_ulong pc
, target_ulong cs_base
, int flags
,
646 TranslationBlock
*tb
;
648 target_ulong phys_pc
, phys_page2
, virt_page2
;
651 phys_pc
= get_phys_addr_code(env
, pc
);
654 /* flush must be done */
656 /* cannot fail at this point */
659 tc_ptr
= code_gen_ptr
;
661 tb
->cs_base
= cs_base
;
664 cpu_gen_code(env
, tb
, &code_gen_size
);
665 code_gen_ptr
= (void *)(((unsigned long)code_gen_ptr
+ code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
667 /* check next page if needed */
668 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
670 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
671 phys_page2
= get_phys_addr_code(env
, virt_page2
);
673 tb_link_phys(tb
, phys_pc
, phys_page2
);
677 /* invalidate all TBs which intersect with the target physical page
678 starting in range [start;end[. NOTE: start and end must refer to
679 the same physical page. 'is_cpu_write_access' should be true if called
680 from a real cpu write access: the virtual CPU will exit the current
681 TB if code is modified inside this TB. */
682 void tb_invalidate_phys_page_range(target_ulong start
, target_ulong end
,
683 int is_cpu_write_access
)
685 int n
, current_tb_modified
, current_tb_not_found
, current_flags
;
686 CPUState
*env
= cpu_single_env
;
688 TranslationBlock
*tb
, *tb_next
, *current_tb
, *saved_tb
;
689 target_ulong tb_start
, tb_end
;
690 target_ulong current_pc
, current_cs_base
;
692 p
= page_find(start
>> TARGET_PAGE_BITS
);
695 if (!p
->code_bitmap
&&
696 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
697 is_cpu_write_access
) {
698 /* build code bitmap */
699 build_page_bitmap(p
);
702 /* we remove all the TBs in the range [start, end[ */
703 /* XXX: see if in some cases it could be faster to invalidate all the code */
704 current_tb_not_found
= is_cpu_write_access
;
705 current_tb_modified
= 0;
706 current_tb
= NULL
; /* avoid warning */
707 current_pc
= 0; /* avoid warning */
708 current_cs_base
= 0; /* avoid warning */
709 current_flags
= 0; /* avoid warning */
713 tb
= (TranslationBlock
*)((long)tb
& ~3);
714 tb_next
= tb
->page_next
[n
];
715 /* NOTE: this is subtle as a TB may span two physical pages */
717 /* NOTE: tb_end may be after the end of the page, but
718 it is not a problem */
719 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
720 tb_end
= tb_start
+ tb
->size
;
722 tb_start
= tb
->page_addr
[1];
723 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
725 if (!(tb_end
<= start
|| tb_start
>= end
)) {
726 #ifdef TARGET_HAS_PRECISE_SMC
727 if (current_tb_not_found
) {
728 current_tb_not_found
= 0;
730 if (env
->mem_write_pc
) {
731 /* now we have a real cpu fault */
732 current_tb
= tb_find_pc(env
->mem_write_pc
);
735 if (current_tb
== tb
&&
736 !(current_tb
->cflags
& CF_SINGLE_INSN
)) {
737 /* If we are modifying the current TB, we must stop
738 its execution. We could be more precise by checking
739 that the modification is after the current PC, but it
740 would require a specialized function to partially
741 restore the CPU state */
743 current_tb_modified
= 1;
744 cpu_restore_state(current_tb
, env
,
745 env
->mem_write_pc
, NULL
);
746 #if defined(TARGET_I386)
747 current_flags
= env
->hflags
;
748 current_flags
|= (env
->eflags
& (IOPL_MASK
| TF_MASK
| VM_MASK
));
749 current_cs_base
= (target_ulong
)env
->segs
[R_CS
].base
;
750 current_pc
= current_cs_base
+ env
->eip
;
752 #error unsupported CPU
755 #endif /* TARGET_HAS_PRECISE_SMC */
756 /* we need to do that to handle the case where a signal
757 occurs while doing tb_phys_invalidate() */
760 saved_tb
= env
->current_tb
;
761 env
->current_tb
= NULL
;
763 tb_phys_invalidate(tb
, -1);
765 env
->current_tb
= saved_tb
;
766 if (env
->interrupt_request
&& env
->current_tb
)
767 cpu_interrupt(env
, env
->interrupt_request
);
772 #if !defined(CONFIG_USER_ONLY)
773 /* if no code remaining, no need to continue to use slow writes */
775 invalidate_page_bitmap(p
);
776 if (is_cpu_write_access
) {
777 tlb_unprotect_code_phys(env
, start
, env
->mem_write_vaddr
);
781 #ifdef TARGET_HAS_PRECISE_SMC
782 if (current_tb_modified
) {
783 /* we generate a block containing just the instruction
784 modifying the memory. It will ensure that it cannot modify
786 env
->current_tb
= NULL
;
787 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
,
789 cpu_resume_from_signal(env
, NULL
);
794 /* len must be <= 8 and start must be a multiple of len */
795 static inline void tb_invalidate_phys_page_fast(target_ulong start
, int len
)
802 fprintf(logfile
, "modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
803 cpu_single_env
->mem_write_vaddr
, len
,
805 cpu_single_env
->eip
+ (long)cpu_single_env
->segs
[R_CS
].base
);
809 p
= page_find(start
>> TARGET_PAGE_BITS
);
812 if (p
->code_bitmap
) {
813 offset
= start
& ~TARGET_PAGE_MASK
;
814 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
815 if (b
& ((1 << len
) - 1))
819 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
823 #if !defined(CONFIG_SOFTMMU)
824 static void tb_invalidate_phys_page(target_ulong addr
,
825 unsigned long pc
, void *puc
)
827 int n
, current_flags
, current_tb_modified
;
828 target_ulong current_pc
, current_cs_base
;
830 TranslationBlock
*tb
, *current_tb
;
831 #ifdef TARGET_HAS_PRECISE_SMC
832 CPUState
*env
= cpu_single_env
;
835 addr
&= TARGET_PAGE_MASK
;
836 p
= page_find(addr
>> TARGET_PAGE_BITS
);
840 current_tb_modified
= 0;
842 current_pc
= 0; /* avoid warning */
843 current_cs_base
= 0; /* avoid warning */
844 current_flags
= 0; /* avoid warning */
845 #ifdef TARGET_HAS_PRECISE_SMC
847 current_tb
= tb_find_pc(pc
);
852 tb
= (TranslationBlock
*)((long)tb
& ~3);
853 #ifdef TARGET_HAS_PRECISE_SMC
854 if (current_tb
== tb
&&
855 !(current_tb
->cflags
& CF_SINGLE_INSN
)) {
856 /* If we are modifying the current TB, we must stop
857 its execution. We could be more precise by checking
858 that the modification is after the current PC, but it
859 would require a specialized function to partially
860 restore the CPU state */
862 current_tb_modified
= 1;
863 cpu_restore_state(current_tb
, env
, pc
, puc
);
864 #if defined(TARGET_I386)
865 current_flags
= env
->hflags
;
866 current_flags
|= (env
->eflags
& (IOPL_MASK
| TF_MASK
| VM_MASK
));
867 current_cs_base
= (target_ulong
)env
->segs
[R_CS
].base
;
868 current_pc
= current_cs_base
+ env
->eip
;
870 #error unsupported CPU
873 #endif /* TARGET_HAS_PRECISE_SMC */
874 tb_phys_invalidate(tb
, addr
);
875 tb
= tb
->page_next
[n
];
878 #ifdef TARGET_HAS_PRECISE_SMC
879 if (current_tb_modified
) {
880 /* we generate a block containing just the instruction
881 modifying the memory. It will ensure that it cannot modify
883 env
->current_tb
= NULL
;
884 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
,
886 cpu_resume_from_signal(env
, puc
);
892 /* add the tb in the target page and protect it if necessary */
893 static inline void tb_alloc_page(TranslationBlock
*tb
,
894 unsigned int n
, target_ulong page_addr
)
897 TranslationBlock
*last_first_tb
;
899 tb
->page_addr
[n
] = page_addr
;
900 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
);
901 tb
->page_next
[n
] = p
->first_tb
;
902 last_first_tb
= p
->first_tb
;
903 p
->first_tb
= (TranslationBlock
*)((long)tb
| n
);
904 invalidate_page_bitmap(p
);
906 #if defined(TARGET_HAS_SMC) || 1
908 #if defined(CONFIG_USER_ONLY)
909 if (p
->flags
& PAGE_WRITE
) {
914 /* force the host page as non writable (writes will have a
915 page fault + mprotect overhead) */
916 page_addr
&= qemu_host_page_mask
;
918 for(addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
919 addr
+= TARGET_PAGE_SIZE
) {
921 p2
= page_find (addr
>> TARGET_PAGE_BITS
);
925 p2
->flags
&= ~PAGE_WRITE
;
926 page_get_flags(addr
);
928 mprotect(g2h(page_addr
), qemu_host_page_size
,
929 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
930 #ifdef DEBUG_TB_INVALIDATE
931 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
936 /* if some code is already present, then the pages are already
937 protected. So we handle the case where only the first TB is
938 allocated in a physical page */
939 if (!last_first_tb
) {
940 tlb_protect_code(page_addr
);
944 #endif /* TARGET_HAS_SMC */
947 /* Allocate a new translation block. Flush the translation buffer if
948 too many translation blocks or too much generated code. */
949 TranslationBlock
*tb_alloc(target_ulong pc
)
951 TranslationBlock
*tb
;
953 if (nb_tbs
>= CODE_GEN_MAX_BLOCKS
||
954 (code_gen_ptr
- code_gen_buffer
) >= CODE_GEN_BUFFER_MAX_SIZE
)
962 /* add a new TB and link it to the physical page tables. phys_page2 is
963 (-1) to indicate that only one page contains the TB. */
964 void tb_link_phys(TranslationBlock
*tb
,
965 target_ulong phys_pc
, target_ulong phys_page2
)
968 TranslationBlock
**ptb
;
970 /* add in the physical hash table */
971 h
= tb_phys_hash_func(phys_pc
);
972 ptb
= &tb_phys_hash
[h
];
973 tb
->phys_hash_next
= *ptb
;
976 /* add in the page list */
977 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
978 if (phys_page2
!= -1)
979 tb_alloc_page(tb
, 1, phys_page2
);
981 tb
->page_addr
[1] = -1;
983 tb
->jmp_first
= (TranslationBlock
*)((long)tb
| 2);
984 tb
->jmp_next
[0] = NULL
;
985 tb
->jmp_next
[1] = NULL
;
987 /* init original jump addresses */
988 if (tb
->tb_next_offset
[0] != 0xffff)
989 tb_reset_jump(tb
, 0);
990 if (tb
->tb_next_offset
[1] != 0xffff)
991 tb_reset_jump(tb
, 1);
993 #ifdef DEBUG_TB_CHECK
998 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
999 tb[1].tc_ptr. Return NULL if not found */
1000 TranslationBlock
*tb_find_pc(unsigned long tc_ptr
)
1002 int m_min
, m_max
, m
;
1004 TranslationBlock
*tb
;
1008 if (tc_ptr
< (unsigned long)code_gen_buffer
||
1009 tc_ptr
>= (unsigned long)code_gen_ptr
)
1011 /* binary search (cf Knuth) */
1014 while (m_min
<= m_max
) {
1015 m
= (m_min
+ m_max
) >> 1;
1017 v
= (unsigned long)tb
->tc_ptr
;
1020 else if (tc_ptr
< v
) {
1029 static void tb_reset_jump_recursive(TranslationBlock
*tb
);
1031 static inline void tb_reset_jump_recursive2(TranslationBlock
*tb
, int n
)
1033 TranslationBlock
*tb1
, *tb_next
, **ptb
;
1036 tb1
= tb
->jmp_next
[n
];
1038 /* find head of list */
1041 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1044 tb1
= tb1
->jmp_next
[n1
];
1046 /* we are now sure now that tb jumps to tb1 */
1049 /* remove tb from the jmp_first list */
1050 ptb
= &tb_next
->jmp_first
;
1054 tb1
= (TranslationBlock
*)((long)tb1
& ~3);
1055 if (n1
== n
&& tb1
== tb
)
1057 ptb
= &tb1
->jmp_next
[n1
];
1059 *ptb
= tb
->jmp_next
[n
];
1060 tb
->jmp_next
[n
] = NULL
;
1062 /* suppress the jump to next tb in generated code */
1063 tb_reset_jump(tb
, n
);
1065 /* suppress jumps in the tb on which we could have jumped */
1066 tb_reset_jump_recursive(tb_next
);
1070 static void tb_reset_jump_recursive(TranslationBlock
*tb
)
1072 tb_reset_jump_recursive2(tb
, 0);
1073 tb_reset_jump_recursive2(tb
, 1);
1076 #if defined(TARGET_HAS_ICE)
1077 static void breakpoint_invalidate(CPUState
*env
, target_ulong pc
)
1079 target_phys_addr_t addr
;
1081 ram_addr_t ram_addr
;
1084 addr
= cpu_get_phys_page_debug(env
, pc
);
1085 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
1087 pd
= IO_MEM_UNASSIGNED
;
1089 pd
= p
->phys_offset
;
1091 ram_addr
= (pd
& TARGET_PAGE_MASK
) | (pc
& ~TARGET_PAGE_MASK
);
1092 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1096 /* Add a watchpoint. */
1097 int cpu_watchpoint_insert(CPUState
*env
, target_ulong addr
)
1101 for (i
= 0; i
< env
->nb_watchpoints
; i
++) {
1102 if (addr
== env
->watchpoint
[i
].vaddr
)
1105 if (env
->nb_watchpoints
>= MAX_WATCHPOINTS
)
1108 i
= env
->nb_watchpoints
++;
1109 env
->watchpoint
[i
].vaddr
= addr
;
1110 tlb_flush_page(env
, addr
);
1111 /* FIXME: This flush is needed because of the hack to make memory ops
1112 terminate the TB. It can be removed once the proper IO trap and
1113 re-execute bits are in. */
1118 /* Remove a watchpoint. */
1119 int cpu_watchpoint_remove(CPUState
*env
, target_ulong addr
)
1123 for (i
= 0; i
< env
->nb_watchpoints
; i
++) {
1124 if (addr
== env
->watchpoint
[i
].vaddr
) {
1125 env
->nb_watchpoints
--;
1126 env
->watchpoint
[i
] = env
->watchpoint
[env
->nb_watchpoints
];
1127 tlb_flush_page(env
, addr
);
1134 /* add a breakpoint. EXCP_DEBUG is returned by the CPU loop if a
1135 breakpoint is reached */
1136 int cpu_breakpoint_insert(CPUState
*env
, target_ulong pc
)
1138 #if defined(TARGET_HAS_ICE)
1141 for(i
= 0; i
< env
->nb_breakpoints
; i
++) {
1142 if (env
->breakpoints
[i
] == pc
)
1146 if (env
->nb_breakpoints
>= MAX_BREAKPOINTS
)
1148 env
->breakpoints
[env
->nb_breakpoints
++] = pc
;
1152 kvm_update_debugger(env
);
1155 breakpoint_invalidate(env
, pc
);
1162 /* remove a breakpoint */
1163 int cpu_breakpoint_remove(CPUState
*env
, target_ulong pc
)
1165 #if defined(TARGET_HAS_ICE)
1167 for(i
= 0; i
< env
->nb_breakpoints
; i
++) {
1168 if (env
->breakpoints
[i
] == pc
)
1173 env
->nb_breakpoints
--;
1174 if (i
< env
->nb_breakpoints
)
1175 env
->breakpoints
[i
] = env
->breakpoints
[env
->nb_breakpoints
];
1179 kvm_update_debugger(env
);
1182 breakpoint_invalidate(env
, pc
);
1189 /* enable or disable single step mode. EXCP_DEBUG is returned by the
1190 CPU loop after each instruction */
1191 void cpu_single_step(CPUState
*env
, int enabled
)
1193 #if defined(TARGET_HAS_ICE)
1194 if (env
->singlestep_enabled
!= enabled
) {
1195 env
->singlestep_enabled
= enabled
;
1196 /* must flush all the translated code to avoid inconsistancies */
1197 /* XXX: only flush what is necessary */
1202 kvm_update_debugger(env
);
1207 /* enable or disable low levels log */
1208 void cpu_set_log(int log_flags
)
1210 loglevel
= log_flags
;
1211 if (loglevel
&& !logfile
) {
1212 logfile
= fopen(logfilename
, log_append
? "a" : "w");
1214 perror(logfilename
);
1217 #if !defined(CONFIG_SOFTMMU)
1218 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1220 static uint8_t logfile_buf
[4096];
1221 setvbuf(logfile
, logfile_buf
, _IOLBF
, sizeof(logfile_buf
));
1224 setvbuf(logfile
, NULL
, _IOLBF
, 0);
1228 if (!loglevel
&& logfile
) {
1234 void cpu_set_log_filename(const char *filename
)
1236 logfilename
= strdup(filename
);
1241 cpu_set_log(loglevel
);
1244 /* mask must never be zero, except for A20 change call */
1245 void cpu_interrupt(CPUState
*env
, int mask
)
1247 TranslationBlock
*tb
;
1248 static int interrupt_lock
;
1250 env
->interrupt_request
|= mask
;
1252 if (kvm_allowed
&& !kvm_irqchip_in_kernel(kvm_context
))
1253 kvm_update_interrupt_request(env
);
1255 /* if the cpu is currently executing code, we must unlink it and
1256 all the potentially executing TB */
1257 tb
= env
->current_tb
;
1258 if (tb
&& !testandset(&interrupt_lock
)) {
1259 env
->current_tb
= NULL
;
1260 tb_reset_jump_recursive(tb
);
1265 void cpu_reset_interrupt(CPUState
*env
, int mask
)
1267 env
->interrupt_request
&= ~mask
;
1270 CPULogItem cpu_log_items
[] = {
1271 { CPU_LOG_TB_OUT_ASM
, "out_asm",
1272 "show generated host assembly code for each compiled TB" },
1273 { CPU_LOG_TB_IN_ASM
, "in_asm",
1274 "show target assembly code for each compiled TB" },
1275 { CPU_LOG_TB_OP
, "op",
1276 "show micro ops for each compiled TB (only usable if 'in_asm' used)" },
1278 { CPU_LOG_TB_OP_OPT
, "op_opt",
1279 "show micro ops after optimization for each compiled TB" },
1281 { CPU_LOG_INT
, "int",
1282 "show interrupts/exceptions in short format" },
1283 { CPU_LOG_EXEC
, "exec",
1284 "show trace before each executed TB (lots of logs)" },
1285 { CPU_LOG_TB_CPU
, "cpu",
1286 "show CPU state before block translation" },
1288 { CPU_LOG_PCALL
, "pcall",
1289 "show protected mode far calls/returns/exceptions" },
1292 { CPU_LOG_IOPORT
, "ioport",
1293 "show all i/o ports accesses" },
1298 static int cmp1(const char *s1
, int n
, const char *s2
)
1300 if (strlen(s2
) != n
)
1302 return memcmp(s1
, s2
, n
) == 0;
1305 /* takes a comma separated list of log masks. Return 0 if error. */
1306 int cpu_str_to_log_mask(const char *str
)
1315 p1
= strchr(p
, ',');
1318 if(cmp1(p
,p1
-p
,"all")) {
1319 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1323 for(item
= cpu_log_items
; item
->mask
!= 0; item
++) {
1324 if (cmp1(p
, p1
- p
, item
->name
))
1338 void cpu_abort(CPUState
*env
, const char *fmt
, ...)
1345 fprintf(stderr
, "qemu: fatal: ");
1346 vfprintf(stderr
, fmt
, ap
);
1347 fprintf(stderr
, "\n");
1349 if(env
->intercept
& INTERCEPT_SVM_MASK
) {
1350 /* most probably the virtual machine should not
1351 be shut down but rather caught by the VMM */
1352 vmexit(SVM_EXIT_SHUTDOWN
, 0);
1354 cpu_dump_state(env
, stderr
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1356 cpu_dump_state(env
, stderr
, fprintf
, 0);
1359 fprintf(logfile
, "qemu: fatal: ");
1360 vfprintf(logfile
, fmt
, ap2
);
1361 fprintf(logfile
, "\n");
1363 cpu_dump_state(env
, logfile
, fprintf
, X86_DUMP_FPU
| X86_DUMP_CCOP
);
1365 cpu_dump_state(env
, logfile
, fprintf
, 0);
1375 CPUState
*cpu_copy(CPUState
*env
)
1377 CPUState
*new_env
= cpu_init(env
->cpu_model_str
);
1378 /* preserve chaining and index */
1379 CPUState
*next_cpu
= new_env
->next_cpu
;
1380 int cpu_index
= new_env
->cpu_index
;
1381 memcpy(new_env
, env
, sizeof(CPUState
));
1382 new_env
->next_cpu
= next_cpu
;
1383 new_env
->cpu_index
= cpu_index
;
1387 #if !defined(CONFIG_USER_ONLY)
1389 /* NOTE: if flush_global is true, also flush global entries (not
1391 void tlb_flush(CPUState
*env
, int flush_global
)
1395 #if defined(DEBUG_TLB)
1396 printf("tlb_flush:\n");
1398 /* must reset current TB so that interrupts cannot modify the
1399 links while we are modifying them */
1400 env
->current_tb
= NULL
;
1402 for(i
= 0; i
< CPU_TLB_SIZE
; i
++) {
1403 env
->tlb_table
[0][i
].addr_read
= -1;
1404 env
->tlb_table
[0][i
].addr_write
= -1;
1405 env
->tlb_table
[0][i
].addr_code
= -1;
1406 env
->tlb_table
[1][i
].addr_read
= -1;
1407 env
->tlb_table
[1][i
].addr_write
= -1;
1408 env
->tlb_table
[1][i
].addr_code
= -1;
1409 #if (NB_MMU_MODES >= 3)
1410 env
->tlb_table
[2][i
].addr_read
= -1;
1411 env
->tlb_table
[2][i
].addr_write
= -1;
1412 env
->tlb_table
[2][i
].addr_code
= -1;
1413 #if (NB_MMU_MODES == 4)
1414 env
->tlb_table
[3][i
].addr_read
= -1;
1415 env
->tlb_table
[3][i
].addr_write
= -1;
1416 env
->tlb_table
[3][i
].addr_code
= -1;
1421 memset (env
->tb_jmp_cache
, 0, TB_JMP_CACHE_SIZE
* sizeof (void *));
1423 #if !defined(CONFIG_SOFTMMU)
1424 munmap((void *)MMAP_AREA_START
, MMAP_AREA_END
- MMAP_AREA_START
);
1427 if (env
->kqemu_enabled
) {
1428 kqemu_flush(env
, flush_global
);
1434 static inline void tlb_flush_entry(CPUTLBEntry
*tlb_entry
, target_ulong addr
)
1436 if (addr
== (tlb_entry
->addr_read
&
1437 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1438 addr
== (tlb_entry
->addr_write
&
1439 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
)) ||
1440 addr
== (tlb_entry
->addr_code
&
1441 (TARGET_PAGE_MASK
| TLB_INVALID_MASK
))) {
1442 tlb_entry
->addr_read
= -1;
1443 tlb_entry
->addr_write
= -1;
1444 tlb_entry
->addr_code
= -1;
1448 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1451 TranslationBlock
*tb
;
1453 #if defined(DEBUG_TLB)
1454 printf("tlb_flush_page: " TARGET_FMT_lx
"\n", addr
);
1456 /* must reset current TB so that interrupts cannot modify the
1457 links while we are modifying them */
1458 env
->current_tb
= NULL
;
1460 addr
&= TARGET_PAGE_MASK
;
1461 i
= (addr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1462 tlb_flush_entry(&env
->tlb_table
[0][i
], addr
);
1463 tlb_flush_entry(&env
->tlb_table
[1][i
], addr
);
1464 #if (NB_MMU_MODES >= 3)
1465 tlb_flush_entry(&env
->tlb_table
[2][i
], addr
);
1466 #if (NB_MMU_MODES == 4)
1467 tlb_flush_entry(&env
->tlb_table
[3][i
], addr
);
1471 /* Discard jump cache entries for any tb which might potentially
1472 overlap the flushed page. */
1473 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1474 memset (&env
->tb_jmp_cache
[i
], 0, TB_JMP_PAGE_SIZE
* sizeof(tb
));
1476 i
= tb_jmp_cache_hash_page(addr
);
1477 memset (&env
->tb_jmp_cache
[i
], 0, TB_JMP_PAGE_SIZE
* sizeof(tb
));
1479 #if !defined(CONFIG_SOFTMMU)
1480 if (addr
< MMAP_AREA_END
)
1481 munmap((void *)addr
, TARGET_PAGE_SIZE
);
1484 if (env
->kqemu_enabled
) {
1485 kqemu_flush_page(env
, addr
);
1490 /* update the TLBs so that writes to code in the virtual page 'addr'
1492 static void tlb_protect_code(ram_addr_t ram_addr
)
1494 cpu_physical_memory_reset_dirty(ram_addr
,
1495 ram_addr
+ TARGET_PAGE_SIZE
,
1499 /* update the TLB so that writes in physical page 'phys_addr' are no longer
1500 tested for self modifying code */
1501 static void tlb_unprotect_code_phys(CPUState
*env
, ram_addr_t ram_addr
,
1504 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] |= CODE_DIRTY_FLAG
;
1507 static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry
,
1508 unsigned long start
, unsigned long length
)
1511 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1512 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1513 if ((addr
- start
) < length
) {
1514 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | IO_MEM_NOTDIRTY
;
1519 void cpu_physical_memory_reset_dirty(ram_addr_t start
, ram_addr_t end
,
1523 unsigned long length
, start1
;
1527 start
&= TARGET_PAGE_MASK
;
1528 end
= TARGET_PAGE_ALIGN(end
);
1530 length
= end
- start
;
1533 len
= length
>> TARGET_PAGE_BITS
;
1535 /* XXX: should not depend on cpu context */
1537 if (env
->kqemu_enabled
) {
1540 for(i
= 0; i
< len
; i
++) {
1541 kqemu_set_notdirty(env
, addr
);
1542 addr
+= TARGET_PAGE_SIZE
;
1546 mask
= ~dirty_flags
;
1547 p
= phys_ram_dirty
+ (start
>> TARGET_PAGE_BITS
);
1548 for(i
= 0; i
< len
; i
++)
1551 /* we modify the TLB cache so that the dirty bit will be set again
1552 when accessing the range */
1553 start1
= start
+ (unsigned long)phys_ram_base
;
1554 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
1555 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1556 tlb_reset_dirty_range(&env
->tlb_table
[0][i
], start1
, length
);
1557 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1558 tlb_reset_dirty_range(&env
->tlb_table
[1][i
], start1
, length
);
1559 #if (NB_MMU_MODES >= 3)
1560 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1561 tlb_reset_dirty_range(&env
->tlb_table
[2][i
], start1
, length
);
1562 #if (NB_MMU_MODES == 4)
1563 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1564 tlb_reset_dirty_range(&env
->tlb_table
[3][i
], start1
, length
);
1569 #if !defined(CONFIG_SOFTMMU)
1570 /* XXX: this is expensive */
1576 for(i
= 0; i
< L1_SIZE
; i
++) {
1579 addr
= i
<< (TARGET_PAGE_BITS
+ L2_BITS
);
1580 for(j
= 0; j
< L2_SIZE
; j
++) {
1581 if (p
->valid_tag
== virt_valid_tag
&&
1582 p
->phys_addr
>= start
&& p
->phys_addr
< end
&&
1583 (p
->prot
& PROT_WRITE
)) {
1584 if (addr
< MMAP_AREA_END
) {
1585 mprotect((void *)addr
, TARGET_PAGE_SIZE
,
1586 p
->prot
& ~PROT_WRITE
);
1589 addr
+= TARGET_PAGE_SIZE
;
1598 int cpu_physical_memory_set_dirty_tracking(int enable
)
1603 r
= kvm_physical_memory_set_dirty_tracking(enable
);
1605 in_migration
= enable
;
1609 int cpu_physical_memory_get_dirty_tracking(void)
1611 return in_migration
;
1614 static inline void tlb_update_dirty(CPUTLBEntry
*tlb_entry
)
1616 ram_addr_t ram_addr
;
1618 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
) {
1619 ram_addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) +
1620 tlb_entry
->addend
- (unsigned long)phys_ram_base
;
1621 if (!cpu_physical_memory_is_dirty(ram_addr
)) {
1622 tlb_entry
->addr_write
|= IO_MEM_NOTDIRTY
;
1627 /* update the TLB according to the current state of the dirty bits */
1628 void cpu_tlb_update_dirty(CPUState
*env
)
1631 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1632 tlb_update_dirty(&env
->tlb_table
[0][i
]);
1633 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1634 tlb_update_dirty(&env
->tlb_table
[1][i
]);
1635 #if (NB_MMU_MODES >= 3)
1636 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1637 tlb_update_dirty(&env
->tlb_table
[2][i
]);
1638 #if (NB_MMU_MODES == 4)
1639 for(i
= 0; i
< CPU_TLB_SIZE
; i
++)
1640 tlb_update_dirty(&env
->tlb_table
[3][i
]);
1645 static inline void tlb_set_dirty1(CPUTLBEntry
*tlb_entry
,
1646 unsigned long start
)
1649 if ((tlb_entry
->addr_write
& ~TARGET_PAGE_MASK
) == IO_MEM_NOTDIRTY
) {
1650 addr
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) + tlb_entry
->addend
;
1651 if (addr
== start
) {
1652 tlb_entry
->addr_write
= (tlb_entry
->addr_write
& TARGET_PAGE_MASK
) | IO_MEM_RAM
;
1657 /* update the TLB corresponding to virtual page vaddr and phys addr
1658 addr so that it is no longer dirty */
1659 static inline void tlb_set_dirty(CPUState
*env
,
1660 unsigned long addr
, target_ulong vaddr
)
1664 addr
&= TARGET_PAGE_MASK
;
1665 i
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1666 tlb_set_dirty1(&env
->tlb_table
[0][i
], addr
);
1667 tlb_set_dirty1(&env
->tlb_table
[1][i
], addr
);
1668 #if (NB_MMU_MODES >= 3)
1669 tlb_set_dirty1(&env
->tlb_table
[2][i
], addr
);
1670 #if (NB_MMU_MODES == 4)
1671 tlb_set_dirty1(&env
->tlb_table
[3][i
], addr
);
1676 /* add a new TLB entry. At most one entry for a given virtual address
1677 is permitted. Return 0 if OK or 2 if the page could not be mapped
1678 (can only happen in non SOFTMMU mode for I/O pages or pages
1679 conflicting with the host address space). */
1680 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
1681 target_phys_addr_t paddr
, int prot
,
1682 int mmu_idx
, int is_softmmu
)
1687 target_ulong address
;
1688 target_phys_addr_t addend
;
1693 p
= phys_page_find(paddr
>> TARGET_PAGE_BITS
);
1695 pd
= IO_MEM_UNASSIGNED
;
1697 pd
= p
->phys_offset
;
1699 #if defined(DEBUG_TLB)
1700 printf("tlb_set_page: vaddr=" TARGET_FMT_lx
" paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1701 vaddr
, (int)paddr
, prot
, mmu_idx
, is_softmmu
, pd
);
1705 #if !defined(CONFIG_SOFTMMU)
1709 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&& !(pd
& IO_MEM_ROMD
)) {
1710 /* IO memory case */
1711 address
= vaddr
| pd
;
1714 /* standard memory */
1716 addend
= (unsigned long)phys_ram_base
+ (pd
& TARGET_PAGE_MASK
);
1719 /* Make accesses to pages with watchpoints go via the
1720 watchpoint trap routines. */
1721 for (i
= 0; i
< env
->nb_watchpoints
; i
++) {
1722 if (vaddr
== (env
->watchpoint
[i
].vaddr
& TARGET_PAGE_MASK
)) {
1723 if (address
& ~TARGET_PAGE_MASK
) {
1724 env
->watchpoint
[i
].addend
= 0;
1725 address
= vaddr
| io_mem_watch
;
1727 env
->watchpoint
[i
].addend
= pd
- paddr
+
1728 (unsigned long) phys_ram_base
;
1729 /* TODO: Figure out how to make read watchpoints coexist
1731 pd
= (pd
& TARGET_PAGE_MASK
) | io_mem_watch
| IO_MEM_ROMD
;
1736 index
= (vaddr
>> TARGET_PAGE_BITS
) & (CPU_TLB_SIZE
- 1);
1738 te
= &env
->tlb_table
[mmu_idx
][index
];
1739 te
->addend
= addend
;
1740 if (prot
& PAGE_READ
) {
1741 te
->addr_read
= address
;
1745 if (prot
& PAGE_EXEC
) {
1746 te
->addr_code
= address
;
1750 if (prot
& PAGE_WRITE
) {
1751 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
1752 (pd
& IO_MEM_ROMD
)) {
1753 /* write access calls the I/O callback */
1754 te
->addr_write
= vaddr
|
1755 (pd
& ~(TARGET_PAGE_MASK
| IO_MEM_ROMD
));
1756 } else if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
1757 !cpu_physical_memory_is_dirty(pd
)) {
1758 te
->addr_write
= vaddr
| IO_MEM_NOTDIRTY
;
1760 te
->addr_write
= address
;
1763 te
->addr_write
= -1;
1766 #if !defined(CONFIG_SOFTMMU)
1768 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
) {
1769 /* IO access: no mapping is done as it will be handled by the
1771 if (!(env
->hflags
& HF_SOFTMMU_MASK
))
1776 if (vaddr
>= MMAP_AREA_END
) {
1779 if (prot
& PROT_WRITE
) {
1780 if ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_ROM
||
1781 #if defined(TARGET_HAS_SMC) || 1
1784 ((pd
& ~TARGET_PAGE_MASK
) == IO_MEM_RAM
&&
1785 !cpu_physical_memory_is_dirty(pd
))) {
1786 /* ROM: we do as if code was inside */
1787 /* if code is present, we only map as read only and save the
1791 vp
= virt_page_find_alloc(vaddr
>> TARGET_PAGE_BITS
, 1);
1794 vp
->valid_tag
= virt_valid_tag
;
1795 prot
&= ~PAGE_WRITE
;
1798 map_addr
= mmap((void *)vaddr
, TARGET_PAGE_SIZE
, prot
,
1799 MAP_SHARED
| MAP_FIXED
, phys_ram_fd
, (pd
& TARGET_PAGE_MASK
));
1800 if (map_addr
== MAP_FAILED
) {
1801 cpu_abort(env
, "mmap failed when mapped physical address 0x%08x to virtual address 0x%08x\n",
1811 /* called from signal handler: invalidate the code and unprotect the
1812 page. Return TRUE if the fault was succesfully handled. */
1813 int page_unprotect(target_ulong addr
, unsigned long pc
, void *puc
)
1815 #if !defined(CONFIG_SOFTMMU)
1818 #if defined(DEBUG_TLB)
1819 printf("page_unprotect: addr=0x%08x\n", addr
);
1821 addr
&= TARGET_PAGE_MASK
;
1823 /* if it is not mapped, no need to worry here */
1824 if (addr
>= MMAP_AREA_END
)
1826 vp
= virt_page_find(addr
>> TARGET_PAGE_BITS
);
1829 /* NOTE: in this case, validate_tag is _not_ tested as it
1830 validates only the code TLB */
1831 if (vp
->valid_tag
!= virt_valid_tag
)
1833 if (!(vp
->prot
& PAGE_WRITE
))
1835 #if defined(DEBUG_TLB)
1836 printf("page_unprotect: addr=0x%08x phys_addr=0x%08x prot=%x\n",
1837 addr
, vp
->phys_addr
, vp
->prot
);
1839 if (mprotect((void *)addr
, TARGET_PAGE_SIZE
, vp
->prot
) < 0)
1840 cpu_abort(cpu_single_env
, "error mprotect addr=0x%lx prot=%d\n",
1841 (unsigned long)addr
, vp
->prot
);
1842 /* set the dirty bit */
1843 phys_ram_dirty
[vp
->phys_addr
>> TARGET_PAGE_BITS
] = 0xff;
1844 /* flush the code inside */
1845 tb_invalidate_phys_page(vp
->phys_addr
, pc
, puc
);
1854 void tlb_flush(CPUState
*env
, int flush_global
)
1858 void tlb_flush_page(CPUState
*env
, target_ulong addr
)
1862 int tlb_set_page_exec(CPUState
*env
, target_ulong vaddr
,
1863 target_phys_addr_t paddr
, int prot
,
1864 int mmu_idx
, int is_softmmu
)
1869 /* dump memory mappings */
1870 void page_dump(FILE *f
)
1872 unsigned long start
, end
;
1873 int i
, j
, prot
, prot1
;
1876 fprintf(f
, "%-8s %-8s %-8s %s\n",
1877 "start", "end", "size", "prot");
1881 for(i
= 0; i
<= L1_SIZE
; i
++) {
1886 for(j
= 0;j
< L2_SIZE
; j
++) {
1891 if (prot1
!= prot
) {
1892 end
= (i
<< (32 - L1_BITS
)) | (j
<< TARGET_PAGE_BITS
);
1894 fprintf(f
, "%08lx-%08lx %08lx %c%c%c\n",
1895 start
, end
, end
- start
,
1896 prot
& PAGE_READ
? 'r' : '-',
1897 prot
& PAGE_WRITE
? 'w' : '-',
1898 prot
& PAGE_EXEC
? 'x' : '-');
1912 int page_get_flags(target_ulong address
)
1916 p
= page_find(address
>> TARGET_PAGE_BITS
);
1922 /* modify the flags of a page and invalidate the code if
1923 necessary. The flag PAGE_WRITE_ORG is positionned automatically
1924 depending on PAGE_WRITE */
1925 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1930 start
= start
& TARGET_PAGE_MASK
;
1931 end
= TARGET_PAGE_ALIGN(end
);
1932 if (flags
& PAGE_WRITE
)
1933 flags
|= PAGE_WRITE_ORG
;
1934 spin_lock(&tb_lock
);
1935 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
1936 p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
);
1937 /* if the write protection is set, then we invalidate the code
1939 if (!(p
->flags
& PAGE_WRITE
) &&
1940 (flags
& PAGE_WRITE
) &&
1942 tb_invalidate_phys_page(addr
, 0, NULL
);
1946 spin_unlock(&tb_lock
);
1949 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1955 end
= TARGET_PAGE_ALIGN(start
+len
); /* must do before we loose bits in the next step */
1956 start
= start
& TARGET_PAGE_MASK
;
1959 /* we've wrapped around */
1961 for(addr
= start
; addr
< end
; addr
+= TARGET_PAGE_SIZE
) {
1962 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1965 if( !(p
->flags
& PAGE_VALID
) )
1968 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
))
1970 if (flags
& PAGE_WRITE
) {
1971 if (!(p
->flags
& PAGE_WRITE_ORG
))
1973 /* unprotect the page if it was put read-only because it
1974 contains translated code */
1975 if (!(p
->flags
& PAGE_WRITE
)) {
1976 if (!page_unprotect(addr
, 0, NULL
))
1985 /* called from signal handler: invalidate the code and unprotect the
1986 page. Return TRUE if the fault was succesfully handled. */
1987 int page_unprotect(target_ulong address
, unsigned long pc
, void *puc
)
1989 unsigned int page_index
, prot
, pindex
;
1991 target_ulong host_start
, host_end
, addr
;
1993 host_start
= address
& qemu_host_page_mask
;
1994 page_index
= host_start
>> TARGET_PAGE_BITS
;
1995 p1
= page_find(page_index
);
1998 host_end
= host_start
+ qemu_host_page_size
;
2001 for(addr
= host_start
;addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2005 /* if the page was really writable, then we change its
2006 protection back to writable */
2007 if (prot
& PAGE_WRITE_ORG
) {
2008 pindex
= (address
- host_start
) >> TARGET_PAGE_BITS
;
2009 if (!(p1
[pindex
].flags
& PAGE_WRITE
)) {
2010 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2011 (prot
& PAGE_BITS
) | PAGE_WRITE
);
2012 p1
[pindex
].flags
|= PAGE_WRITE
;
2013 /* and since the content will be modified, we must invalidate
2014 the corresponding translated code. */
2015 tb_invalidate_phys_page(address
, pc
, puc
);
2016 #ifdef DEBUG_TB_CHECK
2017 tb_invalidate_check(address
);
2025 static inline void tlb_set_dirty(CPUState
*env
,
2026 unsigned long addr
, target_ulong vaddr
)
2029 #endif /* defined(CONFIG_USER_ONLY) */
2031 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2033 static void *subpage_init (target_phys_addr_t base
, uint32_t *phys
,
2035 #define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2038 if (addr > start_addr) \
2041 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2042 if (start_addr2 > 0) \
2046 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
2047 end_addr2 = TARGET_PAGE_SIZE - 1; \
2049 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2050 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2055 /* register physical memory. 'size' must be a multiple of the target
2056 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
2058 void cpu_register_physical_memory(target_phys_addr_t start_addr
,
2060 unsigned long phys_offset
)
2062 target_phys_addr_t addr
, end_addr
;
2065 unsigned long orig_size
= size
;
2068 size
= (size
+ TARGET_PAGE_SIZE
- 1) & TARGET_PAGE_MASK
;
2069 end_addr
= start_addr
+ (target_phys_addr_t
)size
;
2070 for(addr
= start_addr
; addr
!= end_addr
; addr
+= TARGET_PAGE_SIZE
) {
2071 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2072 if (p
&& p
->phys_offset
!= IO_MEM_UNASSIGNED
) {
2073 unsigned long orig_memory
= p
->phys_offset
;
2074 target_phys_addr_t start_addr2
, end_addr2
;
2075 int need_subpage
= 0;
2077 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
, end_addr2
,
2080 if (!(orig_memory
& IO_MEM_SUBPAGE
)) {
2081 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2082 &p
->phys_offset
, orig_memory
);
2084 subpage
= io_mem_opaque
[(orig_memory
& ~TARGET_PAGE_MASK
)
2087 subpage_register(subpage
, start_addr2
, end_addr2
, phys_offset
);
2089 p
->phys_offset
= phys_offset
;
2090 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2091 (phys_offset
& IO_MEM_ROMD
))
2092 phys_offset
+= TARGET_PAGE_SIZE
;
2095 p
= phys_page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2096 p
->phys_offset
= phys_offset
;
2097 if ((phys_offset
& ~TARGET_PAGE_MASK
) <= IO_MEM_ROM
||
2098 (phys_offset
& IO_MEM_ROMD
))
2099 phys_offset
+= TARGET_PAGE_SIZE
;
2101 target_phys_addr_t start_addr2
, end_addr2
;
2102 int need_subpage
= 0;
2104 CHECK_SUBPAGE(addr
, start_addr
, start_addr2
, end_addr
,
2105 end_addr2
, need_subpage
);
2108 subpage
= subpage_init((addr
& TARGET_PAGE_MASK
),
2109 &p
->phys_offset
, IO_MEM_UNASSIGNED
);
2110 subpage_register(subpage
, start_addr2
, end_addr2
,
2117 /* since each CPU stores ram addresses in its TLB cache, we must
2118 reset the modified entries */
2120 for(env
= first_cpu
; env
!= NULL
; env
= env
->next_cpu
) {
2125 /* XXX: temporary until new memory mapping API */
2126 uint32_t cpu_get_physical_page_desc(target_phys_addr_t addr
)
2130 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2132 return IO_MEM_UNASSIGNED
;
2133 return p
->phys_offset
;
2136 /* XXX: better than nothing */
2137 ram_addr_t
qemu_ram_alloc(unsigned long size
)
2140 if ((phys_ram_alloc_offset
+ size
) > phys_ram_size
) {
2141 fprintf(stderr
, "Not enough memory (requested_size = %lu, max memory = %d)\n",
2142 size
, phys_ram_size
);
2145 addr
= phys_ram_alloc_offset
;
2146 phys_ram_alloc_offset
= TARGET_PAGE_ALIGN(phys_ram_alloc_offset
+ size
);
2150 void qemu_ram_free(ram_addr_t addr
)
2154 static uint32_t unassigned_mem_readb(void *opaque
, target_phys_addr_t addr
)
2156 #ifdef DEBUG_UNASSIGNED
2157 printf("Unassigned mem read " TARGET_FMT_plx
"\n", addr
);
2160 do_unassigned_access(addr
, 0, 0, 0);
2162 do_unassigned_access(addr
, 0, 0, 0);
2167 static void unassigned_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2169 #ifdef DEBUG_UNASSIGNED
2170 printf("Unassigned mem write " TARGET_FMT_plx
" = 0x%x\n", addr
, val
);
2173 do_unassigned_access(addr
, 1, 0, 0);
2175 do_unassigned_access(addr
, 1, 0, 0);
2179 static CPUReadMemoryFunc
*unassigned_mem_read
[3] = {
2180 unassigned_mem_readb
,
2181 unassigned_mem_readb
,
2182 unassigned_mem_readb
,
2185 static CPUWriteMemoryFunc
*unassigned_mem_write
[3] = {
2186 unassigned_mem_writeb
,
2187 unassigned_mem_writeb
,
2188 unassigned_mem_writeb
,
2191 static void notdirty_mem_writeb(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2193 unsigned long ram_addr
;
2195 ram_addr
= addr
- (unsigned long)phys_ram_base
;
2196 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2197 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2198 #if !defined(CONFIG_USER_ONLY)
2199 tb_invalidate_phys_page_fast(ram_addr
, 1);
2200 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2203 stb_p((uint8_t *)(long)addr
, val
);
2205 if (cpu_single_env
->kqemu_enabled
&&
2206 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2207 kqemu_modify_page(cpu_single_env
, ram_addr
);
2209 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2210 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2211 /* we remove the notdirty callback only if the code has been
2213 if (dirty_flags
== 0xff)
2214 tlb_set_dirty(cpu_single_env
, addr
, cpu_single_env
->mem_write_vaddr
);
2217 static void notdirty_mem_writew(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2219 unsigned long ram_addr
;
2221 ram_addr
= addr
- (unsigned long)phys_ram_base
;
2222 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2223 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2224 #if !defined(CONFIG_USER_ONLY)
2225 tb_invalidate_phys_page_fast(ram_addr
, 2);
2226 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2229 stw_p((uint8_t *)(long)addr
, val
);
2231 if (cpu_single_env
->kqemu_enabled
&&
2232 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2233 kqemu_modify_page(cpu_single_env
, ram_addr
);
2235 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2236 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2237 /* we remove the notdirty callback only if the code has been
2239 if (dirty_flags
== 0xff)
2240 tlb_set_dirty(cpu_single_env
, addr
, cpu_single_env
->mem_write_vaddr
);
2243 static void notdirty_mem_writel(void *opaque
, target_phys_addr_t addr
, uint32_t val
)
2245 unsigned long ram_addr
;
2247 ram_addr
= addr
- (unsigned long)phys_ram_base
;
2248 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2249 if (!(dirty_flags
& CODE_DIRTY_FLAG
)) {
2250 #if !defined(CONFIG_USER_ONLY)
2251 tb_invalidate_phys_page_fast(ram_addr
, 4);
2252 dirty_flags
= phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
];
2255 stl_p((uint8_t *)(long)addr
, val
);
2257 if (cpu_single_env
->kqemu_enabled
&&
2258 (dirty_flags
& KQEMU_MODIFY_PAGE_MASK
) != KQEMU_MODIFY_PAGE_MASK
)
2259 kqemu_modify_page(cpu_single_env
, ram_addr
);
2261 dirty_flags
|= (0xff & ~CODE_DIRTY_FLAG
);
2262 phys_ram_dirty
[ram_addr
>> TARGET_PAGE_BITS
] = dirty_flags
;
2263 /* we remove the notdirty callback only if the code has been
2265 if (dirty_flags
== 0xff)
2266 tlb_set_dirty(cpu_single_env
, addr
, cpu_single_env
->mem_write_vaddr
);
2269 static CPUReadMemoryFunc
*error_mem_read
[3] = {
2270 NULL
, /* never used */
2271 NULL
, /* never used */
2272 NULL
, /* never used */
2275 static CPUWriteMemoryFunc
*notdirty_mem_write
[3] = {
2276 notdirty_mem_writeb
,
2277 notdirty_mem_writew
,
2278 notdirty_mem_writel
,
2281 #if defined(CONFIG_SOFTMMU)
2282 /* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2283 so these check for a hit then pass through to the normal out-of-line
2285 static uint32_t watch_mem_readb(void *opaque
, target_phys_addr_t addr
)
2287 return ldub_phys(addr
);
2290 static uint32_t watch_mem_readw(void *opaque
, target_phys_addr_t addr
)
2292 return lduw_phys(addr
);
2295 static uint32_t watch_mem_readl(void *opaque
, target_phys_addr_t addr
)
2297 return ldl_phys(addr
);
2300 /* Generate a debug exception if a watchpoint has been hit.
2301 Returns the real physical address of the access. addr will be a host
2302 address in case of a RAM location. */
2303 static target_ulong
check_watchpoint(target_phys_addr_t addr
)
2305 CPUState
*env
= cpu_single_env
;
2307 target_ulong retaddr
;
2311 for (i
= 0; i
< env
->nb_watchpoints
; i
++) {
2312 watch
= env
->watchpoint
[i
].vaddr
;
2313 if (((env
->mem_write_vaddr
^ watch
) & TARGET_PAGE_MASK
) == 0) {
2314 retaddr
= addr
- env
->watchpoint
[i
].addend
;
2315 if (((addr
^ watch
) & ~TARGET_PAGE_MASK
) == 0) {
2316 cpu_single_env
->watchpoint_hit
= i
+ 1;
2317 cpu_interrupt(cpu_single_env
, CPU_INTERRUPT_DEBUG
);
2325 static void watch_mem_writeb(void *opaque
, target_phys_addr_t addr
,
2328 addr
= check_watchpoint(addr
);
2329 stb_phys(addr
, val
);
2332 static void watch_mem_writew(void *opaque
, target_phys_addr_t addr
,
2335 addr
= check_watchpoint(addr
);
2336 stw_phys(addr
, val
);
2339 static void watch_mem_writel(void *opaque
, target_phys_addr_t addr
,
2342 addr
= check_watchpoint(addr
);
2343 stl_phys(addr
, val
);
2346 static CPUReadMemoryFunc
*watch_mem_read
[3] = {
2352 static CPUWriteMemoryFunc
*watch_mem_write
[3] = {
2359 static inline uint32_t subpage_readlen (subpage_t
*mmio
, target_phys_addr_t addr
,
2362 CPUReadMemoryFunc
**mem_read
;
2366 idx
= SUBPAGE_IDX(addr
- mmio
->base
);
2367 #if defined(DEBUG_SUBPAGE)
2368 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d\n", __func__
,
2369 mmio
, len
, addr
, idx
);
2371 mem_read
= mmio
->mem_read
[idx
];
2372 ret
= (*mem_read
[len
])(mmio
->opaque
[idx
], addr
);
2377 static inline void subpage_writelen (subpage_t
*mmio
, target_phys_addr_t addr
,
2378 uint32_t value
, unsigned int len
)
2380 CPUWriteMemoryFunc
**mem_write
;
2383 idx
= SUBPAGE_IDX(addr
- mmio
->base
);
2384 #if defined(DEBUG_SUBPAGE)
2385 printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %08x\n", __func__
,
2386 mmio
, len
, addr
, idx
, value
);
2388 mem_write
= mmio
->mem_write
[idx
];
2389 (*mem_write
[len
])(mmio
->opaque
[idx
], addr
, value
);
2392 static uint32_t subpage_readb (void *opaque
, target_phys_addr_t addr
)
2394 #if defined(DEBUG_SUBPAGE)
2395 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2398 return subpage_readlen(opaque
, addr
, 0);
2401 static void subpage_writeb (void *opaque
, target_phys_addr_t addr
,
2404 #if defined(DEBUG_SUBPAGE)
2405 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2407 subpage_writelen(opaque
, addr
, value
, 0);
2410 static uint32_t subpage_readw (void *opaque
, target_phys_addr_t addr
)
2412 #if defined(DEBUG_SUBPAGE)
2413 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2416 return subpage_readlen(opaque
, addr
, 1);
2419 static void subpage_writew (void *opaque
, target_phys_addr_t addr
,
2422 #if defined(DEBUG_SUBPAGE)
2423 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2425 subpage_writelen(opaque
, addr
, value
, 1);
2428 static uint32_t subpage_readl (void *opaque
, target_phys_addr_t addr
)
2430 #if defined(DEBUG_SUBPAGE)
2431 printf("%s: addr " TARGET_FMT_plx
"\n", __func__
, addr
);
2434 return subpage_readlen(opaque
, addr
, 2);
2437 static void subpage_writel (void *opaque
,
2438 target_phys_addr_t addr
, uint32_t value
)
2440 #if defined(DEBUG_SUBPAGE)
2441 printf("%s: addr " TARGET_FMT_plx
" val %08x\n", __func__
, addr
, value
);
2443 subpage_writelen(opaque
, addr
, value
, 2);
2446 static CPUReadMemoryFunc
*subpage_read
[] = {
2452 static CPUWriteMemoryFunc
*subpage_write
[] = {
2458 static int subpage_register (subpage_t
*mmio
, uint32_t start
, uint32_t end
,
2463 if (start
>= TARGET_PAGE_SIZE
|| end
>= TARGET_PAGE_SIZE
)
2465 idx
= SUBPAGE_IDX(start
);
2466 eidx
= SUBPAGE_IDX(end
);
2467 #if defined(DEBUG_SUBPAGE)
2468 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__
,
2469 mmio
, start
, end
, idx
, eidx
, memory
);
2471 memory
>>= IO_MEM_SHIFT
;
2472 for (; idx
<= eidx
; idx
++) {
2473 mmio
->mem_read
[idx
] = io_mem_read
[memory
];
2474 mmio
->mem_write
[idx
] = io_mem_write
[memory
];
2475 mmio
->opaque
[idx
] = io_mem_opaque
[memory
];
2481 static void *subpage_init (target_phys_addr_t base
, uint32_t *phys
,
2487 mmio
= qemu_mallocz(sizeof(subpage_t
));
2490 subpage_memory
= cpu_register_io_memory(0, subpage_read
, subpage_write
, mmio
);
2491 #if defined(DEBUG_SUBPAGE)
2492 printf("%s: %p base " TARGET_FMT_plx
" len %08x %d\n", __func__
,
2493 mmio
, base
, TARGET_PAGE_SIZE
, subpage_memory
);
2495 *phys
= subpage_memory
| IO_MEM_SUBPAGE
;
2496 subpage_register(mmio
, 0, TARGET_PAGE_SIZE
- 1, orig_memory
);
2502 static void io_mem_init(void)
2504 cpu_register_io_memory(IO_MEM_ROM
>> IO_MEM_SHIFT
, error_mem_read
, unassigned_mem_write
, NULL
);
2505 cpu_register_io_memory(IO_MEM_UNASSIGNED
>> IO_MEM_SHIFT
, unassigned_mem_read
, unassigned_mem_write
, NULL
);
2506 cpu_register_io_memory(IO_MEM_NOTDIRTY
>> IO_MEM_SHIFT
, error_mem_read
, notdirty_mem_write
, NULL
);
2509 #if defined(CONFIG_SOFTMMU)
2510 io_mem_watch
= cpu_register_io_memory(-1, watch_mem_read
,
2511 watch_mem_write
, NULL
);
2513 /* alloc dirty bits array */
2514 phys_ram_dirty
= qemu_vmalloc(phys_ram_size
>> TARGET_PAGE_BITS
);
2515 memset(phys_ram_dirty
, 0xff, phys_ram_size
>> TARGET_PAGE_BITS
);
2518 /* mem_read and mem_write are arrays of functions containing the
2519 function to access byte (index 0), word (index 1) and dword (index
2520 2). All functions must be supplied. If io_index is non zero, the
2521 corresponding io zone is modified. If it is zero, a new io zone is
2522 allocated. The return value can be used with
2523 cpu_register_physical_memory(). (-1) is returned if error. */
2524 int cpu_register_io_memory(int io_index
,
2525 CPUReadMemoryFunc
**mem_read
,
2526 CPUWriteMemoryFunc
**mem_write
,
2531 if (io_index
<= 0) {
2532 if (io_mem_nb
>= IO_MEM_NB_ENTRIES
)
2534 io_index
= io_mem_nb
++;
2536 if (io_index
>= IO_MEM_NB_ENTRIES
)
2540 for(i
= 0;i
< 3; i
++) {
2541 io_mem_read
[io_index
][i
] = mem_read
[i
];
2542 io_mem_write
[io_index
][i
] = mem_write
[i
];
2544 io_mem_opaque
[io_index
] = opaque
;
2545 return io_index
<< IO_MEM_SHIFT
;
2548 CPUWriteMemoryFunc
**cpu_get_io_memory_write(int io_index
)
2550 return io_mem_write
[io_index
>> IO_MEM_SHIFT
];
2553 CPUReadMemoryFunc
**cpu_get_io_memory_read(int io_index
)
2555 return io_mem_read
[io_index
>> IO_MEM_SHIFT
];
2558 /* physical memory access (slow version, mainly for debug) */
2559 #if defined(CONFIG_USER_ONLY)
2560 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2561 int len
, int is_write
)
2568 page
= addr
& TARGET_PAGE_MASK
;
2569 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2572 flags
= page_get_flags(page
);
2573 if (!(flags
& PAGE_VALID
))
2576 if (!(flags
& PAGE_WRITE
))
2578 /* XXX: this code should not depend on lock_user */
2579 if (!(p
= lock_user(VERIFY_WRITE
, addr
, len
, 0)))
2580 /* FIXME - should this return an error rather than just fail? */
2582 memcpy(p
, buf
, len
);
2583 unlock_user(p
, addr
, len
);
2585 if (!(flags
& PAGE_READ
))
2587 /* XXX: this code should not depend on lock_user */
2588 if (!(p
= lock_user(VERIFY_READ
, addr
, len
, 1)))
2589 /* FIXME - should this return an error rather than just fail? */
2591 memcpy(buf
, p
, len
);
2592 unlock_user(p
, addr
, 0);
2601 void cpu_physical_memory_rw(target_phys_addr_t addr
, uint8_t *buf
,
2602 int len
, int is_write
)
2607 target_phys_addr_t page
;
2612 page
= addr
& TARGET_PAGE_MASK
;
2613 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2616 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2618 pd
= IO_MEM_UNASSIGNED
;
2620 pd
= p
->phys_offset
;
2624 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
2625 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2626 /* XXX: could force cpu_single_env to NULL to avoid
2628 if (l
>= 4 && ((addr
& 3) == 0)) {
2629 /* 32 bit write access */
2631 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
2633 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2634 /* 16 bit write access */
2636 io_mem_write
[io_index
][1](io_mem_opaque
[io_index
], addr
, val
);
2639 /* 8 bit write access */
2641 io_mem_write
[io_index
][0](io_mem_opaque
[io_index
], addr
, val
);
2645 unsigned long addr1
;
2646 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2648 ptr
= phys_ram_base
+ addr1
;
2649 memcpy(ptr
, buf
, l
);
2650 if (!cpu_physical_memory_is_dirty(addr1
)) {
2651 /* invalidate code */
2652 tb_invalidate_phys_page_range(addr1
, addr1
+ l
, 0);
2654 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
2655 (0xff & ~CODE_DIRTY_FLAG
);
2658 /* qemu doesn't execute guest code directly, but kvm does
2659 therefore fluch instruction caches */
2660 flush_icache_range((unsigned long)ptr
, ((unsigned long)ptr
)+l
);
2664 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2665 !(pd
& IO_MEM_ROMD
)) {
2667 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2668 if (l
>= 4 && ((addr
& 3) == 0)) {
2669 /* 32 bit read access */
2670 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
2673 } else if (l
>= 2 && ((addr
& 1) == 0)) {
2674 /* 16 bit read access */
2675 val
= io_mem_read
[io_index
][1](io_mem_opaque
[io_index
], addr
);
2679 /* 8 bit read access */
2680 val
= io_mem_read
[io_index
][0](io_mem_opaque
[io_index
], addr
);
2686 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
2687 (addr
& ~TARGET_PAGE_MASK
);
2688 memcpy(buf
, ptr
, l
);
2697 /* used for ROM loading : can write in RAM and ROM */
2698 void cpu_physical_memory_write_rom(target_phys_addr_t addr
,
2699 const uint8_t *buf
, int len
)
2703 target_phys_addr_t page
;
2708 page
= addr
& TARGET_PAGE_MASK
;
2709 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2712 p
= phys_page_find(page
>> TARGET_PAGE_BITS
);
2714 pd
= IO_MEM_UNASSIGNED
;
2716 pd
= p
->phys_offset
;
2719 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
&&
2720 (pd
& ~TARGET_PAGE_MASK
) != IO_MEM_ROM
&&
2721 !(pd
& IO_MEM_ROMD
)) {
2724 unsigned long addr1
;
2725 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2727 ptr
= phys_ram_base
+ addr1
;
2728 memcpy(ptr
, buf
, l
);
2737 /* warning: addr must be aligned */
2738 uint32_t ldl_phys(target_phys_addr_t addr
)
2746 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2748 pd
= IO_MEM_UNASSIGNED
;
2750 pd
= p
->phys_offset
;
2753 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2754 !(pd
& IO_MEM_ROMD
)) {
2756 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2757 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
2760 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
2761 (addr
& ~TARGET_PAGE_MASK
);
2767 /* warning: addr must be aligned */
2768 uint64_t ldq_phys(target_phys_addr_t addr
)
2776 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2778 pd
= IO_MEM_UNASSIGNED
;
2780 pd
= p
->phys_offset
;
2783 if ((pd
& ~TARGET_PAGE_MASK
) > IO_MEM_ROM
&&
2784 !(pd
& IO_MEM_ROMD
)) {
2786 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2787 #ifdef TARGET_WORDS_BIGENDIAN
2788 val
= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
) << 32;
2789 val
|= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4);
2791 val
= io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
);
2792 val
|= (uint64_t)io_mem_read
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4) << 32;
2796 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
2797 (addr
& ~TARGET_PAGE_MASK
);
2804 uint32_t ldub_phys(target_phys_addr_t addr
)
2807 cpu_physical_memory_read(addr
, &val
, 1);
2812 uint32_t lduw_phys(target_phys_addr_t addr
)
2815 cpu_physical_memory_read(addr
, (uint8_t *)&val
, 2);
2816 return tswap16(val
);
2820 #define likely(x) __builtin_expect(!!(x), 1)
2821 #define unlikely(x) __builtin_expect(!!(x), 0)
2824 #define unlikely(x) x
2827 /* warning: addr must be aligned. The ram page is not masked as dirty
2828 and the code inside is not invalidated. It is useful if the dirty
2829 bits are used to track modified PTEs */
2830 void stl_phys_notdirty(target_phys_addr_t addr
, uint32_t val
)
2837 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2839 pd
= IO_MEM_UNASSIGNED
;
2841 pd
= p
->phys_offset
;
2844 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
2845 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2846 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
2848 unsigned long addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2849 ptr
= phys_ram_base
+ addr1
;
2852 if (unlikely(in_migration
)) {
2853 if (!cpu_physical_memory_is_dirty(addr1
)) {
2854 /* invalidate code */
2855 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
2857 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
2858 (0xff & ~CODE_DIRTY_FLAG
);
2864 void stq_phys_notdirty(target_phys_addr_t addr
, uint64_t val
)
2871 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2873 pd
= IO_MEM_UNASSIGNED
;
2875 pd
= p
->phys_offset
;
2878 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
2879 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2880 #ifdef TARGET_WORDS_BIGENDIAN
2881 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
>> 32);
2882 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
);
2884 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
2885 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
+ 4, val
>> 32);
2888 ptr
= phys_ram_base
+ (pd
& TARGET_PAGE_MASK
) +
2889 (addr
& ~TARGET_PAGE_MASK
);
2894 /* warning: addr must be aligned */
2895 void stl_phys(target_phys_addr_t addr
, uint32_t val
)
2902 p
= phys_page_find(addr
>> TARGET_PAGE_BITS
);
2904 pd
= IO_MEM_UNASSIGNED
;
2906 pd
= p
->phys_offset
;
2909 if ((pd
& ~TARGET_PAGE_MASK
) != IO_MEM_RAM
) {
2910 io_index
= (pd
>> IO_MEM_SHIFT
) & (IO_MEM_NB_ENTRIES
- 1);
2911 io_mem_write
[io_index
][2](io_mem_opaque
[io_index
], addr
, val
);
2913 unsigned long addr1
;
2914 addr1
= (pd
& TARGET_PAGE_MASK
) + (addr
& ~TARGET_PAGE_MASK
);
2916 ptr
= phys_ram_base
+ addr1
;
2918 if (!cpu_physical_memory_is_dirty(addr1
)) {
2919 /* invalidate code */
2920 tb_invalidate_phys_page_range(addr1
, addr1
+ 4, 0);
2922 phys_ram_dirty
[addr1
>> TARGET_PAGE_BITS
] |=
2923 (0xff & ~CODE_DIRTY_FLAG
);
2929 void stb_phys(target_phys_addr_t addr
, uint32_t val
)
2932 cpu_physical_memory_write(addr
, &v
, 1);
2936 void stw_phys(target_phys_addr_t addr
, uint32_t val
)
2938 uint16_t v
= tswap16(val
);
2939 cpu_physical_memory_write(addr
, (const uint8_t *)&v
, 2);
2943 void stq_phys(target_phys_addr_t addr
, uint64_t val
)
2946 cpu_physical_memory_write(addr
, (const uint8_t *)&val
, 8);
2951 /* virtual memory access for debug */
2952 int cpu_memory_rw_debug(CPUState
*env
, target_ulong addr
,
2953 uint8_t *buf
, int len
, int is_write
)
2956 target_phys_addr_t phys_addr
;
2960 page
= addr
& TARGET_PAGE_MASK
;
2961 phys_addr
= cpu_get_phys_page_debug(env
, page
);
2962 /* if no physical page mapped, return an error */
2963 if (phys_addr
== -1)
2965 l
= (page
+ TARGET_PAGE_SIZE
) - addr
;
2968 cpu_physical_memory_rw(phys_addr
+ (addr
& ~TARGET_PAGE_MASK
),
2977 void dump_exec_info(FILE *f
,
2978 int (*cpu_fprintf
)(FILE *f
, const char *fmt
, ...))
2980 int i
, target_code_size
, max_target_code_size
;
2981 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
2982 TranslationBlock
*tb
;
2984 target_code_size
= 0;
2985 max_target_code_size
= 0;
2987 direct_jmp_count
= 0;
2988 direct_jmp2_count
= 0;
2989 for(i
= 0; i
< nb_tbs
; i
++) {
2991 target_code_size
+= tb
->size
;
2992 if (tb
->size
> max_target_code_size
)
2993 max_target_code_size
= tb
->size
;
2994 if (tb
->page_addr
[1] != -1)
2996 if (tb
->tb_next_offset
[0] != 0xffff) {
2998 if (tb
->tb_next_offset
[1] != 0xffff) {
2999 direct_jmp2_count
++;
3003 /* XXX: avoid using doubles ? */
3004 cpu_fprintf(f
, "TB count %d\n", nb_tbs
);
3005 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
3006 nb_tbs
? target_code_size
/ nb_tbs
: 0,
3007 max_target_code_size
);
3008 cpu_fprintf(f
, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
3009 nb_tbs
? (code_gen_ptr
- code_gen_buffer
) / nb_tbs
: 0,
3010 target_code_size
? (double) (code_gen_ptr
- code_gen_buffer
) / target_code_size
: 0);
3011 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n",
3013 nb_tbs
? (cross_page
* 100) / nb_tbs
: 0);
3014 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
3016 nb_tbs
? (direct_jmp_count
* 100) / nb_tbs
: 0,
3018 nb_tbs
? (direct_jmp2_count
* 100) / nb_tbs
: 0);
3019 cpu_fprintf(f
, "TB flush count %d\n", tb_flush_count
);
3020 cpu_fprintf(f
, "TB invalidate count %d\n", tb_phys_invalidate_count
);
3021 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
3024 #if !defined(CONFIG_USER_ONLY)
3026 #define MMUSUFFIX _cmmu
3027 #define GETPC() NULL
3028 #define env cpu_single_env
3029 #define SOFTMMU_CODE_ACCESS
3032 #include "softmmu_template.h"
3035 #include "softmmu_template.h"
3038 #include "softmmu_template.h"
3041 #include "softmmu_template.h"