4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include <sys/types.h>
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
36 #include "disas/disas.h"
38 #if defined(CONFIG_USER_ONLY)
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
47 #include <machine/profile.h>
56 #include "exec/address-spaces.h"
59 #include "exec/cputlb.h"
60 #include "translate-all.h"
61 #include "qemu/timer.h"
63 //#define DEBUG_TB_INVALIDATE
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
73 #define SMC_BITMAP_USE_THRESHOLD 10
75 typedef struct PageDesc
{
76 /* list of TBs intersecting this ram page */
77 TranslationBlock
*first_tb
;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count
;
82 #if defined(CONFIG_USER_ONLY)
87 /* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89 #if !defined(CONFIG_USER_ONLY)
90 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
99 /* Size of the L2 (and L3, etc) page tables. */
101 #define V_L2_SIZE (1 << V_L2_BITS)
103 /* The bits remaining after N lower levels of page tables. */
104 #define V_L1_BITS_REM \
105 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
107 #if V_L1_BITS_REM < 4
108 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
110 #define V_L1_BITS V_L1_BITS_REM
113 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
115 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
117 uintptr_t qemu_real_host_page_size
;
118 uintptr_t qemu_host_page_size
;
119 uintptr_t qemu_host_page_mask
;
121 /* This is a multi-level map on the virtual address space.
122 The bottom level has pointers to PageDesc. */
123 static void *l1_map
[V_L1_SIZE
];
125 /* code generation context */
128 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
129 tb_page_addr_t phys_page2
);
130 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
132 void cpu_gen_init(void)
134 tcg_context_init(&tcg_ctx
);
137 /* return non zero if the very first instruction is invalid so that
138 the virtual CPU can trigger an exception.
140 '*gen_code_size_ptr' contains the size of the generated code (host
143 int cpu_gen_code(CPUArchState
*env
, TranslationBlock
*tb
, int *gen_code_size_ptr
)
145 TCGContext
*s
= &tcg_ctx
;
146 uint8_t *gen_code_buf
;
148 #ifdef CONFIG_PROFILER
152 #ifdef CONFIG_PROFILER
153 s
->tb_count1
++; /* includes aborted translations because of
155 ti
= profile_getclock();
159 gen_intermediate_code(env
, tb
);
161 /* generate machine code */
162 gen_code_buf
= tb
->tc_ptr
;
163 tb
->tb_next_offset
[0] = 0xffff;
164 tb
->tb_next_offset
[1] = 0xffff;
165 s
->tb_next_offset
= tb
->tb_next_offset
;
166 #ifdef USE_DIRECT_JUMP
167 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
170 s
->tb_jmp_offset
= NULL
;
171 s
->tb_next
= tb
->tb_next
;
174 #ifdef CONFIG_PROFILER
176 s
->interm_time
+= profile_getclock() - ti
;
177 s
->code_time
-= profile_getclock();
179 gen_code_size
= tcg_gen_code(s
, gen_code_buf
);
180 *gen_code_size_ptr
= gen_code_size
;
181 #ifdef CONFIG_PROFILER
182 s
->code_time
+= profile_getclock();
183 s
->code_in_len
+= tb
->size
;
184 s
->code_out_len
+= gen_code_size
;
188 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
189 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr
);
190 log_disas(tb
->tc_ptr
, *gen_code_size_ptr
);
198 /* The cpu state corresponding to 'searched_pc' is restored.
200 static int cpu_restore_state_from_tb(TranslationBlock
*tb
, CPUArchState
*env
,
201 uintptr_t searched_pc
)
203 CPUState
*cpu
= ENV_GET_CPU(env
);
204 TCGContext
*s
= &tcg_ctx
;
207 #ifdef CONFIG_PROFILER
211 #ifdef CONFIG_PROFILER
212 ti
= profile_getclock();
216 gen_intermediate_code_pc(env
, tb
);
219 /* Reset the cycle counter to the start of the block. */
220 env
->icount_decr
.u16
.low
+= tb
->icount
;
221 /* Clear the IO flag. */
225 /* find opc index corresponding to search_pc */
226 tc_ptr
= (uintptr_t)tb
->tc_ptr
;
227 if (searched_pc
< tc_ptr
)
230 s
->tb_next_offset
= tb
->tb_next_offset
;
231 #ifdef USE_DIRECT_JUMP
232 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
235 s
->tb_jmp_offset
= NULL
;
236 s
->tb_next
= tb
->tb_next
;
238 j
= tcg_gen_code_search_pc(s
, (uint8_t *)tc_ptr
, searched_pc
- tc_ptr
);
241 /* now find start of instruction before */
242 while (s
->gen_opc_instr_start
[j
] == 0) {
245 env
->icount_decr
.u16
.low
-= s
->gen_opc_icount
[j
];
247 restore_state_to_opc(env
, tb
, j
);
249 #ifdef CONFIG_PROFILER
250 s
->restore_time
+= profile_getclock() - ti
;
256 bool cpu_restore_state(CPUArchState
*env
, uintptr_t retaddr
)
258 TranslationBlock
*tb
;
260 tb
= tb_find_pc(retaddr
);
262 cpu_restore_state_from_tb(tb
, env
, retaddr
);
269 static inline void map_exec(void *addr
, long size
)
272 VirtualProtect(addr
, size
,
273 PAGE_EXECUTE_READWRITE
, &old_protect
);
276 static inline void map_exec(void *addr
, long size
)
278 unsigned long start
, end
, page_size
;
280 page_size
= getpagesize();
281 start
= (unsigned long)addr
;
282 start
&= ~(page_size
- 1);
284 end
= (unsigned long)addr
+ size
;
285 end
+= page_size
- 1;
286 end
&= ~(page_size
- 1);
288 mprotect((void *)start
, end
- start
,
289 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
293 void page_size_init(void)
295 /* NOTE: we can always suppose that qemu_host_page_size >=
298 SYSTEM_INFO system_info
;
300 GetSystemInfo(&system_info
);
301 qemu_real_host_page_size
= system_info
.dwPageSize
;
303 qemu_real_host_page_size
= getpagesize();
305 if (qemu_host_page_size
== 0) {
306 qemu_host_page_size
= qemu_real_host_page_size
;
308 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
309 qemu_host_page_size
= TARGET_PAGE_SIZE
;
311 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
314 static void page_init(void)
317 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
319 #ifdef HAVE_KINFO_GETVMMAP
320 struct kinfo_vmentry
*freep
;
323 freep
= kinfo_getvmmap(getpid(), &cnt
);
326 for (i
= 0; i
< cnt
; i
++) {
327 unsigned long startaddr
, endaddr
;
329 startaddr
= freep
[i
].kve_start
;
330 endaddr
= freep
[i
].kve_end
;
331 if (h2g_valid(startaddr
)) {
332 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
334 if (h2g_valid(endaddr
)) {
335 endaddr
= h2g(endaddr
);
336 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
338 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
340 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
351 last_brk
= (unsigned long)sbrk(0);
353 f
= fopen("/compat/linux/proc/self/maps", "r");
358 unsigned long startaddr
, endaddr
;
361 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
363 if (n
== 2 && h2g_valid(startaddr
)) {
364 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
366 if (h2g_valid(endaddr
)) {
367 endaddr
= h2g(endaddr
);
371 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
383 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
389 #if defined(CONFIG_USER_ONLY)
390 /* We can't use g_malloc because it may recurse into a locked mutex. */
391 # define ALLOC(P, SIZE) \
393 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
394 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
397 # define ALLOC(P, SIZE) \
398 do { P = g_malloc0(SIZE); } while (0)
401 /* Level 1. Always allocated. */
402 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
405 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
412 ALLOC(p
, sizeof(void *) * V_L2_SIZE
);
416 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
424 ALLOC(pd
, sizeof(PageDesc
) * V_L2_SIZE
);
430 return pd
+ (index
& (V_L2_SIZE
- 1));
433 static inline PageDesc
*page_find(tb_page_addr_t index
)
435 return page_find_alloc(index
, 0);
438 #if !defined(CONFIG_USER_ONLY)
439 #define mmap_lock() do { } while (0)
440 #define mmap_unlock() do { } while (0)
443 #if defined(CONFIG_USER_ONLY)
444 /* Currently it is not recommended to allocate big chunks of data in
445 user mode. It will change when a dedicated libc will be used. */
446 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
447 region in which the guest needs to run. Revisit this. */
448 #define USE_STATIC_CODE_GEN_BUFFER
451 /* ??? Should configure for this, not list operating systems here. */
452 #if (defined(__linux__) \
453 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
454 || defined(__DragonFly__) || defined(__OpenBSD__) \
455 || defined(__NetBSD__))
459 /* Minimum size of the code gen buffer. This number is randomly chosen,
460 but not so small that we can't have a fair number of TB's live. */
461 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
463 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
464 indicated, this is constrained by the range of direct branches on the
465 host cpu, as used by the TCG implementation of goto_tb. */
466 #if defined(__x86_64__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
468 #elif defined(__sparc__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
470 #elif defined(__aarch64__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
472 #elif defined(__arm__)
473 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
474 #elif defined(__s390x__)
475 /* We have a +- 4GB range on the branches; leave some slop. */
476 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
478 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
484 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
485 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
487 static inline size_t size_code_gen_buffer(size_t tb_size
)
489 /* Size the buffer. */
491 #ifdef USE_STATIC_CODE_GEN_BUFFER
492 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
494 /* ??? Needs adjustments. */
495 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
496 static buffer, we could size this on RESERVED_VA, on the text
497 segment size of the executable, or continue to use the default. */
498 tb_size
= (unsigned long)(ram_size
/ 4);
501 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
502 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
504 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
505 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
507 tcg_ctx
.code_gen_buffer_size
= tb_size
;
511 #ifdef USE_STATIC_CODE_GEN_BUFFER
512 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
513 __attribute__((aligned(CODE_GEN_ALIGN
)));
515 static inline void *alloc_code_gen_buffer(void)
517 map_exec(static_code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
);
518 return static_code_gen_buffer
;
520 #elif defined(USE_MMAP)
521 static inline void *alloc_code_gen_buffer(void)
523 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
527 /* Constrain the position of the buffer based on the host cpu.
528 Note that these addresses are chosen in concert with the
529 addresses assigned in the relevant linker script file. */
530 # if defined(__PIE__) || defined(__PIC__)
531 /* Don't bother setting a preferred location if we're building
532 a position-independent executable. We're more likely to get
533 an address near the main executable if we let the kernel
534 choose the address. */
535 # elif defined(__x86_64__) && defined(MAP_32BIT)
536 /* Force the memory down into low memory with the executable.
537 Leave the choice of exact location with the kernel. */
539 /* Cannot expect to map more than 800MB in low memory. */
540 if (tcg_ctx
.code_gen_buffer_size
> 800u * 1024 * 1024) {
541 tcg_ctx
.code_gen_buffer_size
= 800u * 1024 * 1024;
543 # elif defined(__sparc__)
544 start
= 0x40000000ul
;
545 # elif defined(__s390x__)
546 start
= 0x90000000ul
;
549 buf
= mmap((void *)start
, tcg_ctx
.code_gen_buffer_size
,
550 PROT_WRITE
| PROT_READ
| PROT_EXEC
, flags
, -1, 0);
551 return buf
== MAP_FAILED
? NULL
: buf
;
554 static inline void *alloc_code_gen_buffer(void)
556 void *buf
= g_malloc(tcg_ctx
.code_gen_buffer_size
);
559 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
563 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
565 static inline void code_gen_alloc(size_t tb_size
)
567 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
568 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
569 if (tcg_ctx
.code_gen_buffer
== NULL
) {
570 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
574 qemu_madvise(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
,
577 /* Steal room for the prologue at the end of the buffer. This ensures
578 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
579 from TB's to the prologue are going to be in range. It also means
580 that we don't need to mark (additional) portions of the data segment
582 tcg_ctx
.code_gen_prologue
= tcg_ctx
.code_gen_buffer
+
583 tcg_ctx
.code_gen_buffer_size
- 1024;
584 tcg_ctx
.code_gen_buffer_size
-= 1024;
586 tcg_ctx
.code_gen_buffer_max_size
= tcg_ctx
.code_gen_buffer_size
-
587 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
588 tcg_ctx
.code_gen_max_blocks
= tcg_ctx
.code_gen_buffer_size
/
589 CODE_GEN_AVG_BLOCK_SIZE
;
591 g_malloc(tcg_ctx
.code_gen_max_blocks
* sizeof(TranslationBlock
));
594 /* Must be called before using the QEMU cpus. 'tb_size' is the size
595 (in bytes) allocated to the translation buffer. Zero means default
597 void tcg_exec_init(unsigned long tb_size
)
600 code_gen_alloc(tb_size
);
601 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
602 tcg_register_jit(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
);
604 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
605 /* There's no guest base to take into account, so go ahead and
606 initialize the prologue now. */
607 tcg_prologue_init(&tcg_ctx
);
611 bool tcg_enabled(void)
613 return tcg_ctx
.code_gen_buffer
!= NULL
;
616 /* Allocate a new translation block. Flush the translation buffer if
617 too many translation blocks or too much generated code. */
618 static TranslationBlock
*tb_alloc(target_ulong pc
)
620 TranslationBlock
*tb
;
622 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
||
623 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) >=
624 tcg_ctx
.code_gen_buffer_max_size
) {
627 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
633 void tb_free(TranslationBlock
*tb
)
635 /* In practice this is mostly used for single use temporary TB
636 Ignore the hard cases and just back up if this TB happens to
637 be the last one generated. */
638 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
639 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
640 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
641 tcg_ctx
.tb_ctx
.nb_tbs
--;
645 static inline void invalidate_page_bitmap(PageDesc
*p
)
647 if (p
->code_bitmap
) {
648 g_free(p
->code_bitmap
);
649 p
->code_bitmap
= NULL
;
651 p
->code_write_count
= 0;
654 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
655 static void page_flush_tb_1(int level
, void **lp
)
665 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
666 pd
[i
].first_tb
= NULL
;
667 invalidate_page_bitmap(pd
+ i
);
672 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
673 page_flush_tb_1(level
- 1, pp
+ i
);
678 static void page_flush_tb(void)
682 for (i
= 0; i
< V_L1_SIZE
; i
++) {
683 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
687 /* flush all the translation blocks */
688 /* XXX: tb_flush is currently not thread safe */
689 void tb_flush(CPUArchState
*env1
)
693 #if defined(DEBUG_FLUSH)
694 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
695 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
696 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
697 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
698 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
700 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
701 > tcg_ctx
.code_gen_buffer_size
) {
702 cpu_abort(env1
, "Internal error: code buffer overflow\n");
704 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
707 CPUArchState
*env
= cpu
->env_ptr
;
709 memset(env
->tb_jmp_cache
, 0, sizeof(env
->tb_jmp_cache
));
712 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
715 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
716 /* XXX: flush processor icache at this point if cache flush is
718 tcg_ctx
.tb_ctx
.tb_flush_count
++;
721 #ifdef DEBUG_TB_CHECK
723 static void tb_invalidate_check(target_ulong address
)
725 TranslationBlock
*tb
;
728 address
&= TARGET_PAGE_MASK
;
729 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
730 for (tb
= tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
731 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
732 address
>= tb
->pc
+ tb
->size
)) {
733 printf("ERROR invalidate: address=" TARGET_FMT_lx
734 " PC=%08lx size=%04x\n",
735 address
, (long)tb
->pc
, tb
->size
);
741 /* verify that all the pages have correct rights for code */
742 static void tb_page_check(void)
744 TranslationBlock
*tb
;
745 int i
, flags1
, flags2
;
747 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
748 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
749 tb
= tb
->phys_hash_next
) {
750 flags1
= page_get_flags(tb
->pc
);
751 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
752 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
753 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
754 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
762 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
764 TranslationBlock
*tb1
;
769 *ptb
= tb1
->phys_hash_next
;
772 ptb
= &tb1
->phys_hash_next
;
776 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
778 TranslationBlock
*tb1
;
783 n1
= (uintptr_t)tb1
& 3;
784 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
786 *ptb
= tb1
->page_next
[n1
];
789 ptb
= &tb1
->page_next
[n1
];
793 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
795 TranslationBlock
*tb1
, **ptb
;
798 ptb
= &tb
->jmp_next
[n
];
801 /* find tb(n) in circular list */
804 n1
= (uintptr_t)tb1
& 3;
805 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
806 if (n1
== n
&& tb1
== tb
) {
810 ptb
= &tb1
->jmp_first
;
812 ptb
= &tb1
->jmp_next
[n1
];
815 /* now we can suppress tb(n) from the list */
816 *ptb
= tb
->jmp_next
[n
];
818 tb
->jmp_next
[n
] = NULL
;
822 /* reset the jump entry 'n' of a TB so that it is not chained to
824 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
826 tb_set_jmp_target(tb
, n
, (uintptr_t)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
829 /* invalidate one TB */
830 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
835 tb_page_addr_t phys_pc
;
836 TranslationBlock
*tb1
, *tb2
;
838 /* remove the TB from the hash list */
839 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
840 h
= tb_phys_hash_func(phys_pc
);
841 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
843 /* remove the TB from the page list */
844 if (tb
->page_addr
[0] != page_addr
) {
845 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
846 tb_page_remove(&p
->first_tb
, tb
);
847 invalidate_page_bitmap(p
);
849 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
850 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
851 tb_page_remove(&p
->first_tb
, tb
);
852 invalidate_page_bitmap(p
);
855 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
857 /* remove the TB from the hash list */
858 h
= tb_jmp_cache_hash_func(tb
->pc
);
860 CPUArchState
*env
= cpu
->env_ptr
;
862 if (env
->tb_jmp_cache
[h
] == tb
) {
863 env
->tb_jmp_cache
[h
] = NULL
;
867 /* suppress this TB from the two jump lists */
868 tb_jmp_remove(tb
, 0);
869 tb_jmp_remove(tb
, 1);
871 /* suppress any remaining jumps to this TB */
874 n1
= (uintptr_t)tb1
& 3;
878 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
879 tb2
= tb1
->jmp_next
[n1
];
880 tb_reset_jump(tb1
, n1
);
881 tb1
->jmp_next
[n1
] = NULL
;
884 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2); /* fail safe */
886 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
889 static inline void set_bits(uint8_t *tab
, int start
, int len
)
895 mask
= 0xff << (start
& 7);
896 if ((start
& ~7) == (end
& ~7)) {
898 mask
&= ~(0xff << (end
& 7));
903 start
= (start
+ 8) & ~7;
905 while (start
< end1
) {
910 mask
= ~(0xff << (end
& 7));
916 static void build_page_bitmap(PageDesc
*p
)
918 int n
, tb_start
, tb_end
;
919 TranslationBlock
*tb
;
921 p
->code_bitmap
= g_malloc0(TARGET_PAGE_SIZE
/ 8);
925 n
= (uintptr_t)tb
& 3;
926 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
927 /* NOTE: this is subtle as a TB may span two physical pages */
929 /* NOTE: tb_end may be after the end of the page, but
930 it is not a problem */
931 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
932 tb_end
= tb_start
+ tb
->size
;
933 if (tb_end
> TARGET_PAGE_SIZE
) {
934 tb_end
= TARGET_PAGE_SIZE
;
938 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
940 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
941 tb
= tb
->page_next
[n
];
945 TranslationBlock
*tb_gen_code(CPUArchState
*env
,
946 target_ulong pc
, target_ulong cs_base
,
947 int flags
, int cflags
)
949 TranslationBlock
*tb
;
951 tb_page_addr_t phys_pc
, phys_page2
;
952 target_ulong virt_page2
;
955 phys_pc
= get_page_addr_code(env
, pc
);
958 /* flush must be done */
960 /* cannot fail at this point */
962 /* Don't forget to invalidate previous TB info. */
963 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
965 tc_ptr
= tcg_ctx
.code_gen_ptr
;
967 tb
->cs_base
= cs_base
;
970 cpu_gen_code(env
, tb
, &code_gen_size
);
971 tcg_ctx
.code_gen_ptr
= (void *)(((uintptr_t)tcg_ctx
.code_gen_ptr
+
972 code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
974 /* check next page if needed */
975 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
977 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
978 phys_page2
= get_page_addr_code(env
, virt_page2
);
980 tb_link_page(tb
, phys_pc
, phys_page2
);
985 * Invalidate all TBs which intersect with the target physical address range
986 * [start;end[. NOTE: start and end may refer to *different* physical pages.
987 * 'is_cpu_write_access' should be true if called from a real cpu write
988 * access: the virtual CPU will exit the current TB if code is modified inside
991 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
,
992 int is_cpu_write_access
)
994 while (start
< end
) {
995 tb_invalidate_phys_page_range(start
, end
, is_cpu_write_access
);
996 start
&= TARGET_PAGE_MASK
;
997 start
+= TARGET_PAGE_SIZE
;
1002 * Invalidate all TBs which intersect with the target physical address range
1003 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1004 * 'is_cpu_write_access' should be true if called from a real cpu write
1005 * access: the virtual CPU will exit the current TB if code is modified inside
1008 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1009 int is_cpu_write_access
)
1011 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1012 CPUState
*cpu
= current_cpu
;
1013 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1014 CPUArchState
*env
= NULL
;
1016 tb_page_addr_t tb_start
, tb_end
;
1019 #ifdef TARGET_HAS_PRECISE_SMC
1020 int current_tb_not_found
= is_cpu_write_access
;
1021 TranslationBlock
*current_tb
= NULL
;
1022 int current_tb_modified
= 0;
1023 target_ulong current_pc
= 0;
1024 target_ulong current_cs_base
= 0;
1025 int current_flags
= 0;
1026 #endif /* TARGET_HAS_PRECISE_SMC */
1028 p
= page_find(start
>> TARGET_PAGE_BITS
);
1032 if (!p
->code_bitmap
&&
1033 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1034 is_cpu_write_access
) {
1035 /* build code bitmap */
1036 build_page_bitmap(p
);
1038 #if defined(TARGET_HAS_PRECISE_SMC) || !defined(CONFIG_USER_ONLY)
1044 /* we remove all the TBs in the range [start, end[ */
1045 /* XXX: see if in some cases it could be faster to invalidate all
1048 while (tb
!= NULL
) {
1049 n
= (uintptr_t)tb
& 3;
1050 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1051 tb_next
= tb
->page_next
[n
];
1052 /* NOTE: this is subtle as a TB may span two physical pages */
1054 /* NOTE: tb_end may be after the end of the page, but
1055 it is not a problem */
1056 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1057 tb_end
= tb_start
+ tb
->size
;
1059 tb_start
= tb
->page_addr
[1];
1060 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1062 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1063 #ifdef TARGET_HAS_PRECISE_SMC
1064 if (current_tb_not_found
) {
1065 current_tb_not_found
= 0;
1067 if (cpu
->mem_io_pc
) {
1068 /* now we have a real cpu fault */
1069 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1072 if (current_tb
== tb
&&
1073 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1074 /* If we are modifying the current TB, we must stop
1075 its execution. We could be more precise by checking
1076 that the modification is after the current PC, but it
1077 would require a specialized function to partially
1078 restore the CPU state */
1080 current_tb_modified
= 1;
1081 cpu_restore_state_from_tb(current_tb
, env
, cpu
->mem_io_pc
);
1082 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1085 #endif /* TARGET_HAS_PRECISE_SMC */
1086 /* we need to do that to handle the case where a signal
1087 occurs while doing tb_phys_invalidate() */
1090 saved_tb
= cpu
->current_tb
;
1091 cpu
->current_tb
= NULL
;
1093 tb_phys_invalidate(tb
, -1);
1095 cpu
->current_tb
= saved_tb
;
1096 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1097 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1103 #if !defined(CONFIG_USER_ONLY)
1104 /* if no code remaining, no need to continue to use slow writes */
1106 invalidate_page_bitmap(p
);
1107 if (is_cpu_write_access
) {
1108 tlb_unprotect_code_phys(env
, start
, cpu
->mem_io_vaddr
);
1112 #ifdef TARGET_HAS_PRECISE_SMC
1113 if (current_tb_modified
) {
1114 /* we generate a block containing just the instruction
1115 modifying the memory. It will ensure that it cannot modify
1117 cpu
->current_tb
= NULL
;
1118 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1119 cpu_resume_from_signal(env
, NULL
);
1124 /* len must be <= 8 and start must be a multiple of len */
1125 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1132 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1133 cpu_single_env
->mem_io_vaddr
, len
,
1134 cpu_single_env
->eip
,
1135 cpu_single_env
->eip
+
1136 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1139 p
= page_find(start
>> TARGET_PAGE_BITS
);
1143 if (p
->code_bitmap
) {
1144 offset
= start
& ~TARGET_PAGE_MASK
;
1145 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1146 if (b
& ((1 << len
) - 1)) {
1151 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1155 #if !defined(CONFIG_SOFTMMU)
1156 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1157 uintptr_t pc
, void *puc
,
1160 TranslationBlock
*tb
;
1163 #ifdef TARGET_HAS_PRECISE_SMC
1164 TranslationBlock
*current_tb
= NULL
;
1165 CPUState
*cpu
= current_cpu
;
1166 CPUArchState
*env
= NULL
;
1167 int current_tb_modified
= 0;
1168 target_ulong current_pc
= 0;
1169 target_ulong current_cs_base
= 0;
1170 int current_flags
= 0;
1173 addr
&= TARGET_PAGE_MASK
;
1174 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1179 #ifdef TARGET_HAS_PRECISE_SMC
1180 if (tb
&& pc
!= 0) {
1181 current_tb
= tb_find_pc(pc
);
1187 while (tb
!= NULL
) {
1188 n
= (uintptr_t)tb
& 3;
1189 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1190 #ifdef TARGET_HAS_PRECISE_SMC
1191 if (current_tb
== tb
&&
1192 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1193 /* If we are modifying the current TB, we must stop
1194 its execution. We could be more precise by checking
1195 that the modification is after the current PC, but it
1196 would require a specialized function to partially
1197 restore the CPU state */
1199 current_tb_modified
= 1;
1200 cpu_restore_state_from_tb(current_tb
, env
, pc
);
1201 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1204 #endif /* TARGET_HAS_PRECISE_SMC */
1205 tb_phys_invalidate(tb
, addr
);
1206 tb
= tb
->page_next
[n
];
1209 #ifdef TARGET_HAS_PRECISE_SMC
1210 if (current_tb_modified
) {
1211 /* we generate a block containing just the instruction
1212 modifying the memory. It will ensure that it cannot modify
1214 cpu
->current_tb
= NULL
;
1215 tb_gen_code(env
, current_pc
, current_cs_base
, current_flags
, 1);
1219 cpu_resume_from_signal(env
, puc
);
1225 /* add the tb in the target page and protect it if necessary */
1226 static inline void tb_alloc_page(TranslationBlock
*tb
,
1227 unsigned int n
, tb_page_addr_t page_addr
)
1230 #ifndef CONFIG_USER_ONLY
1231 bool page_already_protected
;
1234 tb
->page_addr
[n
] = page_addr
;
1235 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1236 tb
->page_next
[n
] = p
->first_tb
;
1237 #ifndef CONFIG_USER_ONLY
1238 page_already_protected
= p
->first_tb
!= NULL
;
1240 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1241 invalidate_page_bitmap(p
);
1243 #if defined(TARGET_HAS_SMC) || 1
1245 #if defined(CONFIG_USER_ONLY)
1246 if (p
->flags
& PAGE_WRITE
) {
1251 /* force the host page as non writable (writes will have a
1252 page fault + mprotect overhead) */
1253 page_addr
&= qemu_host_page_mask
;
1255 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1256 addr
+= TARGET_PAGE_SIZE
) {
1258 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1263 p2
->flags
&= ~PAGE_WRITE
;
1265 mprotect(g2h(page_addr
), qemu_host_page_size
,
1266 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1267 #ifdef DEBUG_TB_INVALIDATE
1268 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1273 /* if some code is already present, then the pages are already
1274 protected. So we handle the case where only the first TB is
1275 allocated in a physical page */
1276 if (!page_already_protected
) {
1277 tlb_protect_code(page_addr
);
1281 #endif /* TARGET_HAS_SMC */
1284 /* add a new TB and link it to the physical page tables. phys_page2 is
1285 (-1) to indicate that only one page contains the TB. */
1286 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1287 tb_page_addr_t phys_page2
)
1290 TranslationBlock
**ptb
;
1292 /* Grab the mmap lock to stop another thread invalidating this TB
1293 before we are done. */
1295 /* add in the physical hash table */
1296 h
= tb_phys_hash_func(phys_pc
);
1297 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1298 tb
->phys_hash_next
= *ptb
;
1301 /* add in the page list */
1302 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1303 if (phys_page2
!= -1) {
1304 tb_alloc_page(tb
, 1, phys_page2
);
1306 tb
->page_addr
[1] = -1;
1309 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2);
1310 tb
->jmp_next
[0] = NULL
;
1311 tb
->jmp_next
[1] = NULL
;
1313 /* init original jump addresses */
1314 if (tb
->tb_next_offset
[0] != 0xffff) {
1315 tb_reset_jump(tb
, 0);
1317 if (tb
->tb_next_offset
[1] != 0xffff) {
1318 tb_reset_jump(tb
, 1);
1321 #ifdef DEBUG_TB_CHECK
1327 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1328 tb[1].tc_ptr. Return NULL if not found */
1329 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1331 int m_min
, m_max
, m
;
1333 TranslationBlock
*tb
;
1335 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1338 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1339 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1342 /* binary search (cf Knuth) */
1344 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1345 while (m_min
<= m_max
) {
1346 m
= (m_min
+ m_max
) >> 1;
1347 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1348 v
= (uintptr_t)tb
->tc_ptr
;
1351 } else if (tc_ptr
< v
) {
1357 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1360 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1361 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1363 ram_addr_t ram_addr
;
1367 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1368 if (!(memory_region_is_ram(mr
)
1369 || memory_region_is_romd(mr
))) {
1372 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1374 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1376 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1378 void tb_check_watchpoint(CPUArchState
*env
)
1380 CPUState
*cpu
= ENV_GET_CPU(env
);
1381 TranslationBlock
*tb
;
1383 tb
= tb_find_pc(cpu
->mem_io_pc
);
1385 cpu_abort(env
, "check_watchpoint: could not find TB for pc=%p",
1386 (void *)cpu
->mem_io_pc
);
1388 cpu_restore_state_from_tb(tb
, env
, cpu
->mem_io_pc
);
1389 tb_phys_invalidate(tb
, -1);
1392 #ifndef CONFIG_USER_ONLY
1393 /* mask must never be zero, except for A20 change call */
1394 static void tcg_handle_interrupt(CPUState
*cpu
, int mask
)
1396 CPUArchState
*env
= cpu
->env_ptr
;
1399 old_mask
= cpu
->interrupt_request
;
1400 cpu
->interrupt_request
|= mask
;
1403 * If called from iothread context, wake the target cpu in
1406 if (!qemu_cpu_is_self(cpu
)) {
1412 env
->icount_decr
.u16
.high
= 0xffff;
1413 if (!cpu_can_do_io(cpu
)
1414 && (mask
& ~old_mask
) != 0) {
1415 cpu_abort(env
, "Raised interrupt while not in I/O function");
1418 cpu
->tcg_exit_req
= 1;
1422 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1424 /* in deterministic execution mode, instructions doing device I/Os
1425 must be at the end of the TB */
1426 void cpu_io_recompile(CPUArchState
*env
, uintptr_t retaddr
)
1428 TranslationBlock
*tb
;
1430 target_ulong pc
, cs_base
;
1433 tb
= tb_find_pc(retaddr
);
1435 cpu_abort(env
, "cpu_io_recompile: could not find TB for pc=%p",
1438 n
= env
->icount_decr
.u16
.low
+ tb
->icount
;
1439 cpu_restore_state_from_tb(tb
, env
, retaddr
);
1440 /* Calculate how many instructions had been executed before the fault
1442 n
= n
- env
->icount_decr
.u16
.low
;
1443 /* Generate a new TB ending on the I/O insn. */
1445 /* On MIPS and SH, delay slot instructions can only be restarted if
1446 they were already the first instruction in the TB. If this is not
1447 the first instruction in a TB then re-execute the preceding
1449 #if defined(TARGET_MIPS)
1450 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1451 env
->active_tc
.PC
-= 4;
1452 env
->icount_decr
.u16
.low
++;
1453 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1455 #elif defined(TARGET_SH4)
1456 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1459 env
->icount_decr
.u16
.low
++;
1460 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1463 /* This should never happen. */
1464 if (n
> CF_COUNT_MASK
) {
1465 cpu_abort(env
, "TB too big during recompile");
1468 cflags
= n
| CF_LAST_IO
;
1470 cs_base
= tb
->cs_base
;
1472 tb_phys_invalidate(tb
, -1);
1473 /* FIXME: In theory this could raise an exception. In practice
1474 we have already translated the block once so it's probably ok. */
1475 tb_gen_code(env
, pc
, cs_base
, flags
, cflags
);
1476 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1477 the first in the TB) then we end up generating a whole new TB and
1478 repeating the fault, which is horribly inefficient.
1479 Better would be to execute just this insn uncached, or generate a
1481 cpu_resume_from_signal(env
, NULL
);
1484 void tb_flush_jmp_cache(CPUArchState
*env
, target_ulong addr
)
1488 /* Discard jump cache entries for any tb which might potentially
1489 overlap the flushed page. */
1490 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1491 memset(&env
->tb_jmp_cache
[i
], 0,
1492 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1494 i
= tb_jmp_cache_hash_page(addr
);
1495 memset(&env
->tb_jmp_cache
[i
], 0,
1496 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1499 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1501 int i
, target_code_size
, max_target_code_size
;
1502 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1503 TranslationBlock
*tb
;
1505 target_code_size
= 0;
1506 max_target_code_size
= 0;
1508 direct_jmp_count
= 0;
1509 direct_jmp2_count
= 0;
1510 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1511 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1512 target_code_size
+= tb
->size
;
1513 if (tb
->size
> max_target_code_size
) {
1514 max_target_code_size
= tb
->size
;
1516 if (tb
->page_addr
[1] != -1) {
1519 if (tb
->tb_next_offset
[0] != 0xffff) {
1521 if (tb
->tb_next_offset
[1] != 0xffff) {
1522 direct_jmp2_count
++;
1526 /* XXX: avoid using doubles ? */
1527 cpu_fprintf(f
, "Translation buffer state:\n");
1528 cpu_fprintf(f
, "gen code size %td/%zd\n",
1529 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1530 tcg_ctx
.code_gen_buffer_max_size
);
1531 cpu_fprintf(f
, "TB count %d/%d\n",
1532 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1533 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1534 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1535 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1536 max_target_code_size
);
1537 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1538 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1539 tcg_ctx
.code_gen_buffer
) /
1540 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1541 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1542 tcg_ctx
.code_gen_buffer
) /
1543 target_code_size
: 0);
1544 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1545 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1546 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1547 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1549 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1550 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1552 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1553 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1554 cpu_fprintf(f
, "\nStatistics:\n");
1555 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1556 cpu_fprintf(f
, "TB invalidate count %d\n",
1557 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1558 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1559 tcg_dump_info(f
, cpu_fprintf
);
1562 #else /* CONFIG_USER_ONLY */
1564 void cpu_interrupt(CPUState
*cpu
, int mask
)
1566 cpu
->interrupt_request
|= mask
;
1567 cpu
->tcg_exit_req
= 1;
1571 * Walks guest process memory "regions" one by one
1572 * and calls callback function 'fn' for each region.
1574 struct walk_memory_regions_data
{
1575 walk_memory_regions_fn fn
;
1581 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1582 abi_ulong end
, int new_prot
)
1584 if (data
->start
!= -1ul) {
1585 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1591 data
->start
= (new_prot
? end
: -1ul);
1592 data
->prot
= new_prot
;
1597 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1598 abi_ulong base
, int level
, void **lp
)
1604 return walk_memory_regions_end(data
, base
, 0);
1610 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1611 int prot
= pd
[i
].flags
;
1613 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1614 if (prot
!= data
->prot
) {
1615 rc
= walk_memory_regions_end(data
, pa
, prot
);
1624 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1625 pa
= base
| ((abi_ulong
)i
<<
1626 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1627 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1637 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1639 struct walk_memory_regions_data data
;
1647 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1648 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
1649 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1656 return walk_memory_regions_end(&data
, 0, 0);
1659 static int dump_region(void *priv
, abi_ulong start
,
1660 abi_ulong end
, unsigned long prot
)
1662 FILE *f
= (FILE *)priv
;
1664 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
1665 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
1666 start
, end
, end
- start
,
1667 ((prot
& PAGE_READ
) ? 'r' : '-'),
1668 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1669 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1674 /* dump memory mappings */
1675 void page_dump(FILE *f
)
1677 const int length
= sizeof(abi_ulong
) * 2;
1678 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1679 length
, "start", length
, "end", length
, "size", "prot");
1680 walk_memory_regions(f
, dump_region
);
1683 int page_get_flags(target_ulong address
)
1687 p
= page_find(address
>> TARGET_PAGE_BITS
);
1694 /* Modify the flags of a page and invalidate the code if necessary.
1695 The flag PAGE_WRITE_ORG is positioned automatically depending
1696 on PAGE_WRITE. The mmap_lock should already be held. */
1697 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1699 target_ulong addr
, len
;
1701 /* This function should never be called with addresses outside the
1702 guest address space. If this assert fires, it probably indicates
1703 a missing call to h2g_valid. */
1704 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1705 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1707 assert(start
< end
);
1709 start
= start
& TARGET_PAGE_MASK
;
1710 end
= TARGET_PAGE_ALIGN(end
);
1712 if (flags
& PAGE_WRITE
) {
1713 flags
|= PAGE_WRITE_ORG
;
1716 for (addr
= start
, len
= end
- start
;
1718 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1719 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1721 /* If the write protection bit is set, then we invalidate
1723 if (!(p
->flags
& PAGE_WRITE
) &&
1724 (flags
& PAGE_WRITE
) &&
1726 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1732 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1738 /* This function should never be called with addresses outside the
1739 guest address space. If this assert fires, it probably indicates
1740 a missing call to h2g_valid. */
1741 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1742 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1748 if (start
+ len
- 1 < start
) {
1749 /* We've wrapped around. */
1753 /* must do before we loose bits in the next step */
1754 end
= TARGET_PAGE_ALIGN(start
+ len
);
1755 start
= start
& TARGET_PAGE_MASK
;
1757 for (addr
= start
, len
= end
- start
;
1759 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1760 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1764 if (!(p
->flags
& PAGE_VALID
)) {
1768 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1771 if (flags
& PAGE_WRITE
) {
1772 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1775 /* unprotect the page if it was put read-only because it
1776 contains translated code */
1777 if (!(p
->flags
& PAGE_WRITE
)) {
1778 if (!page_unprotect(addr
, 0, NULL
)) {
1788 /* called from signal handler: invalidate the code and unprotect the
1789 page. Return TRUE if the fault was successfully handled. */
1790 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1794 target_ulong host_start
, host_end
, addr
;
1796 /* Technically this isn't safe inside a signal handler. However we
1797 know this only ever happens in a synchronous SEGV handler, so in
1798 practice it seems to be ok. */
1801 p
= page_find(address
>> TARGET_PAGE_BITS
);
1807 /* if the page was really writable, then we change its
1808 protection back to writable */
1809 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
1810 host_start
= address
& qemu_host_page_mask
;
1811 host_end
= host_start
+ qemu_host_page_size
;
1814 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
1815 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1816 p
->flags
|= PAGE_WRITE
;
1819 /* and since the content will be modified, we must invalidate
1820 the corresponding translated code. */
1821 tb_invalidate_phys_page(addr
, pc
, puc
, true);
1822 #ifdef DEBUG_TB_CHECK
1823 tb_invalidate_check(addr
);
1826 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
1835 #endif /* CONFIG_USER_ONLY */