4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
30 #include "qemu-common.h"
31 #define NO_CPU_IO_DEFS
34 #include "disas/disas.h"
36 #if defined(CONFIG_USER_ONLY)
38 #if defined(TARGET_X86_64)
41 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42 #include <sys/param.h>
43 #if __FreeBSD_version >= 700104
44 #define HAVE_KINFO_GETVMMAP
45 #define sigqueue sigqueue_freebsd /* avoid redefinition */
48 #include <machine/profile.h>
57 #include "exec/address-spaces.h"
60 #include "exec/cputlb.h"
61 #include "exec/tb-hash.h"
62 #include "translate-all.h"
63 #include "qemu/bitmap.h"
64 #include "qemu/timer.h"
66 //#define DEBUG_TB_INVALIDATE
68 /* make various TB consistency checks */
69 //#define DEBUG_TB_CHECK
71 #if !defined(CONFIG_USER_ONLY)
72 /* TB consistency checks only implemented for usermode emulation. */
76 #define SMC_BITMAP_USE_THRESHOLD 10
78 typedef struct PageDesc
{
79 /* list of TBs intersecting this ram page */
80 TranslationBlock
*first_tb
;
81 /* in order to optimize self modifying code, we count the number
82 of lookups we do to a given page to use a bitmap */
83 unsigned int code_write_count
;
84 unsigned long *code_bitmap
;
85 #if defined(CONFIG_USER_ONLY)
90 /* In system mode we want L1_MAP to be based on ram offsets,
91 while in user mode we want it to be based on virtual addresses. */
92 #if !defined(CONFIG_USER_ONLY)
93 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
94 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
99 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
102 /* Size of the L2 (and L3, etc) page tables. */
104 #define V_L2_SIZE (1 << V_L2_BITS)
106 /* The bits remaining after N lower levels of page tables. */
107 #define V_L1_BITS_REM \
108 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
110 #if V_L1_BITS_REM < 4
111 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
113 #define V_L1_BITS V_L1_BITS_REM
116 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
118 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
120 uintptr_t qemu_host_page_size
;
121 uintptr_t qemu_host_page_mask
;
123 /* The bottom level has pointers to PageDesc */
124 static void *l1_map
[V_L1_SIZE
];
126 /* code generation context */
129 /* translation block context */
130 #ifdef CONFIG_USER_ONLY
131 __thread
int have_tb_lock
;
136 #ifdef CONFIG_USER_ONLY
137 assert(!have_tb_lock
);
138 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
145 #ifdef CONFIG_USER_ONLY
146 assert(have_tb_lock
);
148 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
152 void tb_lock_reset(void)
154 #ifdef CONFIG_USER_ONLY
156 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
162 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
163 tb_page_addr_t phys_page2
);
164 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
166 void cpu_gen_init(void)
168 tcg_context_init(&tcg_ctx
);
171 /* return non zero if the very first instruction is invalid so that
172 * the virtual CPU can trigger an exception.
174 * '*gen_code_size_ptr' contains the size of the generated code (host
177 * Called with mmap_lock held for user-mode emulation.
179 int cpu_gen_code(CPUArchState
*env
, TranslationBlock
*tb
, int *gen_code_size_ptr
)
181 TCGContext
*s
= &tcg_ctx
;
182 tcg_insn_unit
*gen_code_buf
;
184 #ifdef CONFIG_PROFILER
188 #ifdef CONFIG_PROFILER
189 s
->tb_count1
++; /* includes aborted translations because of
191 ti
= profile_getclock();
195 gen_intermediate_code(env
, tb
);
197 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
199 /* generate machine code */
200 gen_code_buf
= tb
->tc_ptr
;
201 tb
->tb_next_offset
[0] = 0xffff;
202 tb
->tb_next_offset
[1] = 0xffff;
203 s
->tb_next_offset
= tb
->tb_next_offset
;
204 #ifdef USE_DIRECT_JUMP
205 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
208 s
->tb_jmp_offset
= NULL
;
209 s
->tb_next
= tb
->tb_next
;
212 #ifdef CONFIG_PROFILER
214 s
->interm_time
+= profile_getclock() - ti
;
215 s
->code_time
-= profile_getclock();
217 gen_code_size
= tcg_gen_code(s
, gen_code_buf
);
218 *gen_code_size_ptr
= gen_code_size
;
219 #ifdef CONFIG_PROFILER
220 s
->code_time
+= profile_getclock();
221 s
->code_in_len
+= tb
->size
;
222 s
->code_out_len
+= gen_code_size
;
226 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
227 qemu_log("OUT: [size=%d]\n", gen_code_size
);
228 log_disas(tb
->tc_ptr
, gen_code_size
);
236 /* The cpu state corresponding to 'searched_pc' is restored.
238 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
239 uintptr_t searched_pc
)
241 CPUArchState
*env
= cpu
->env_ptr
;
242 TCGContext
*s
= &tcg_ctx
;
245 #ifdef CONFIG_PROFILER
249 #ifdef CONFIG_PROFILER
250 ti
= profile_getclock();
254 gen_intermediate_code_pc(env
, tb
);
256 if (tb
->cflags
& CF_USE_ICOUNT
) {
258 /* Reset the cycle counter to the start of the block. */
259 cpu
->icount_decr
.u16
.low
+= tb
->icount
;
260 /* Clear the IO flag. */
264 /* find opc index corresponding to search_pc */
265 tc_ptr
= (uintptr_t)tb
->tc_ptr
;
266 if (searched_pc
< tc_ptr
)
269 s
->tb_next_offset
= tb
->tb_next_offset
;
270 #ifdef USE_DIRECT_JUMP
271 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
274 s
->tb_jmp_offset
= NULL
;
275 s
->tb_next
= tb
->tb_next
;
277 j
= tcg_gen_code_search_pc(s
, (tcg_insn_unit
*)tc_ptr
,
278 searched_pc
- tc_ptr
);
281 /* now find start of instruction before */
282 while (s
->gen_opc_instr_start
[j
] == 0) {
285 cpu
->icount_decr
.u16
.low
-= s
->gen_opc_icount
[j
];
287 restore_state_to_opc(env
, tb
, j
);
289 #ifdef CONFIG_PROFILER
290 s
->restore_time
+= profile_getclock() - ti
;
296 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
298 TranslationBlock
*tb
;
300 tb
= tb_find_pc(retaddr
);
302 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
303 if (tb
->cflags
& CF_NOCACHE
) {
304 /* one-shot translation, invalidate it immediately */
305 cpu
->current_tb
= NULL
;
306 tb_phys_invalidate(tb
, -1);
315 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
318 VirtualProtect(addr
, size
,
319 PAGE_EXECUTE_READWRITE
, &old_protect
);
322 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
324 unsigned long start
, end
, page_size
;
326 page_size
= getpagesize();
327 start
= (unsigned long)addr
;
328 start
&= ~(page_size
- 1);
330 end
= (unsigned long)addr
+ size
;
331 end
+= page_size
- 1;
332 end
&= ~(page_size
- 1);
334 mprotect((void *)start
, end
- start
,
335 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
339 void page_size_init(void)
341 /* NOTE: we can always suppose that qemu_host_page_size >=
343 qemu_real_host_page_size
= getpagesize();
344 qemu_real_host_page_mask
= ~(qemu_real_host_page_size
- 1);
345 if (qemu_host_page_size
== 0) {
346 qemu_host_page_size
= qemu_real_host_page_size
;
348 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
349 qemu_host_page_size
= TARGET_PAGE_SIZE
;
351 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
354 static void page_init(void)
357 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
359 #ifdef HAVE_KINFO_GETVMMAP
360 struct kinfo_vmentry
*freep
;
363 freep
= kinfo_getvmmap(getpid(), &cnt
);
366 for (i
= 0; i
< cnt
; i
++) {
367 unsigned long startaddr
, endaddr
;
369 startaddr
= freep
[i
].kve_start
;
370 endaddr
= freep
[i
].kve_end
;
371 if (h2g_valid(startaddr
)) {
372 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
374 if (h2g_valid(endaddr
)) {
375 endaddr
= h2g(endaddr
);
376 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
378 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
380 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
391 last_brk
= (unsigned long)sbrk(0);
393 f
= fopen("/compat/linux/proc/self/maps", "r");
398 unsigned long startaddr
, endaddr
;
401 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
403 if (n
== 2 && h2g_valid(startaddr
)) {
404 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
406 if (h2g_valid(endaddr
)) {
407 endaddr
= h2g(endaddr
);
411 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
424 * Called with mmap_lock held for user-mode emulation.
426 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
432 /* Level 1. Always allocated. */
433 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
436 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
437 void **p
= atomic_rcu_read(lp
);
443 p
= g_new0(void *, V_L2_SIZE
);
444 atomic_rcu_set(lp
, p
);
447 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
450 pd
= atomic_rcu_read(lp
);
455 pd
= g_new0(PageDesc
, V_L2_SIZE
);
456 atomic_rcu_set(lp
, pd
);
459 return pd
+ (index
& (V_L2_SIZE
- 1));
462 static inline PageDesc
*page_find(tb_page_addr_t index
)
464 return page_find_alloc(index
, 0);
467 #if defined(CONFIG_USER_ONLY)
468 /* Currently it is not recommended to allocate big chunks of data in
469 user mode. It will change when a dedicated libc will be used. */
470 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
471 region in which the guest needs to run. Revisit this. */
472 #define USE_STATIC_CODE_GEN_BUFFER
475 /* ??? Should configure for this, not list operating systems here. */
476 #if (defined(__linux__) \
477 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
478 || defined(__DragonFly__) || defined(__OpenBSD__) \
479 || defined(__NetBSD__))
483 /* Minimum size of the code gen buffer. This number is randomly chosen,
484 but not so small that we can't have a fair number of TB's live. */
485 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
487 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
488 indicated, this is constrained by the range of direct branches on the
489 host cpu, as used by the TCG implementation of goto_tb. */
490 #if defined(__x86_64__)
491 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
492 #elif defined(__sparc__)
493 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
494 #elif defined(__aarch64__)
495 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
496 #elif defined(__arm__)
497 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
498 #elif defined(__s390x__)
499 /* We have a +- 4GB range on the branches; leave some slop. */
500 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
501 #elif defined(__mips__)
502 /* We have a 256MB branch region, but leave room to make sure the
503 main executable is also within that region. */
504 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
506 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
509 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
511 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
512 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
513 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
515 static inline size_t size_code_gen_buffer(size_t tb_size
)
517 /* Size the buffer. */
519 #ifdef USE_STATIC_CODE_GEN_BUFFER
520 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
522 /* ??? Needs adjustments. */
523 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
524 static buffer, we could size this on RESERVED_VA, on the text
525 segment size of the executable, or continue to use the default. */
526 tb_size
= (unsigned long)(ram_size
/ 4);
529 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
530 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
532 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
533 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
535 tcg_ctx
.code_gen_buffer_size
= tb_size
;
540 /* In order to use J and JAL within the code_gen_buffer, we require
541 that the buffer not cross a 256MB boundary. */
542 static inline bool cross_256mb(void *addr
, size_t size
)
544 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & 0xf0000000;
547 /* We weren't able to allocate a buffer without crossing that boundary,
548 so make do with the larger portion of the buffer that doesn't cross.
549 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
550 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
552 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & 0xf0000000);
553 size_t size2
= buf1
+ size1
- buf2
;
561 tcg_ctx
.code_gen_buffer_size
= size1
;
566 #ifdef USE_STATIC_CODE_GEN_BUFFER
567 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
568 __attribute__((aligned(CODE_GEN_ALIGN
)));
570 static inline void *alloc_code_gen_buffer(void)
572 void *buf
= static_code_gen_buffer
;
574 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
575 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
578 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
581 #elif defined(USE_MMAP)
582 static inline void *alloc_code_gen_buffer(void)
584 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
588 /* Constrain the position of the buffer based on the host cpu.
589 Note that these addresses are chosen in concert with the
590 addresses assigned in the relevant linker script file. */
591 # if defined(__PIE__) || defined(__PIC__)
592 /* Don't bother setting a preferred location if we're building
593 a position-independent executable. We're more likely to get
594 an address near the main executable if we let the kernel
595 choose the address. */
596 # elif defined(__x86_64__) && defined(MAP_32BIT)
597 /* Force the memory down into low memory with the executable.
598 Leave the choice of exact location with the kernel. */
600 /* Cannot expect to map more than 800MB in low memory. */
601 if (tcg_ctx
.code_gen_buffer_size
> 800u * 1024 * 1024) {
602 tcg_ctx
.code_gen_buffer_size
= 800u * 1024 * 1024;
604 # elif defined(__sparc__)
605 start
= 0x40000000ul
;
606 # elif defined(__s390x__)
607 start
= 0x90000000ul
;
608 # elif defined(__mips__)
609 /* ??? We ought to more explicitly manage layout for softmmu too. */
610 # ifdef CONFIG_USER_ONLY
611 start
= 0x68000000ul
;
612 # elif _MIPS_SIM == _ABI64
613 start
= 0x128000000ul
;
615 start
= 0x08000000ul
;
619 buf
= mmap((void *)start
, tcg_ctx
.code_gen_buffer_size
,
620 PROT_WRITE
| PROT_READ
| PROT_EXEC
, flags
, -1, 0);
621 if (buf
== MAP_FAILED
) {
626 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
627 /* Try again, with the original still mapped, to avoid re-acquiring
628 that 256mb crossing. This time don't specify an address. */
629 size_t size2
, size1
= tcg_ctx
.code_gen_buffer_size
;
630 void *buf2
= mmap(NULL
, size1
, PROT_WRITE
| PROT_READ
| PROT_EXEC
,
632 if (buf2
!= MAP_FAILED
) {
633 if (!cross_256mb(buf2
, size1
)) {
634 /* Success! Use the new buffer. */
638 /* Failure. Work with what we had. */
642 /* Split the original buffer. Free the smaller half. */
643 buf2
= split_cross_256mb(buf
, size1
);
644 size2
= tcg_ctx
.code_gen_buffer_size
;
645 munmap(buf
+ (buf
== buf2
? size2
: 0), size1
- size2
);
653 static inline void *alloc_code_gen_buffer(void)
655 void *buf
= g_try_malloc(tcg_ctx
.code_gen_buffer_size
);
662 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
663 void *buf2
= g_malloc(tcg_ctx
.code_gen_buffer_size
);
664 if (buf2
!= NULL
&& !cross_256mb(buf2
, size1
)) {
665 /* Success! Use the new buffer. */
669 /* Failure. Work with what we had. Since this is malloc
670 and not mmap, we can't free the other half. */
672 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
677 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
680 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
682 static inline void code_gen_alloc(size_t tb_size
)
684 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
685 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
686 if (tcg_ctx
.code_gen_buffer
== NULL
) {
687 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
691 qemu_madvise(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
,
694 /* Steal room for the prologue at the end of the buffer. This ensures
695 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
696 from TB's to the prologue are going to be in range. It also means
697 that we don't need to mark (additional) portions of the data segment
699 tcg_ctx
.code_gen_prologue
= tcg_ctx
.code_gen_buffer
+
700 tcg_ctx
.code_gen_buffer_size
- 1024;
701 tcg_ctx
.code_gen_buffer_size
-= 1024;
703 tcg_ctx
.code_gen_buffer_max_size
= tcg_ctx
.code_gen_buffer_size
-
704 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
705 tcg_ctx
.code_gen_max_blocks
= tcg_ctx
.code_gen_buffer_size
/
706 CODE_GEN_AVG_BLOCK_SIZE
;
708 g_malloc(tcg_ctx
.code_gen_max_blocks
* sizeof(TranslationBlock
));
709 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
712 /* Must be called before using the QEMU cpus. 'tb_size' is the size
713 (in bytes) allocated to the translation buffer. Zero means default
715 void tcg_exec_init(uintptr_t tb_size
)
718 code_gen_alloc(tb_size
);
719 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
720 tcg_register_jit(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
);
722 #if defined(CONFIG_SOFTMMU)
723 /* There's no guest base to take into account, so go ahead and
724 initialize the prologue now. */
725 tcg_prologue_init(&tcg_ctx
);
729 bool tcg_enabled(void)
731 return tcg_ctx
.code_gen_buffer
!= NULL
;
734 /* Allocate a new translation block. Flush the translation buffer if
735 too many translation blocks or too much generated code. */
736 static TranslationBlock
*tb_alloc(target_ulong pc
)
738 TranslationBlock
*tb
;
740 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
||
741 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) >=
742 tcg_ctx
.code_gen_buffer_max_size
) {
745 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
751 void tb_free(TranslationBlock
*tb
)
753 /* In practice this is mostly used for single use temporary TB
754 Ignore the hard cases and just back up if this TB happens to
755 be the last one generated. */
756 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
757 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
758 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
759 tcg_ctx
.tb_ctx
.nb_tbs
--;
763 static inline void invalidate_page_bitmap(PageDesc
*p
)
765 g_free(p
->code_bitmap
);
766 p
->code_bitmap
= NULL
;
767 p
->code_write_count
= 0;
770 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
771 static void page_flush_tb_1(int level
, void **lp
)
781 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
782 pd
[i
].first_tb
= NULL
;
783 invalidate_page_bitmap(pd
+ i
);
788 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
789 page_flush_tb_1(level
- 1, pp
+ i
);
794 static void page_flush_tb(void)
798 for (i
= 0; i
< V_L1_SIZE
; i
++) {
799 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
803 /* flush all the translation blocks */
804 /* XXX: tb_flush is currently not thread safe */
805 void tb_flush(CPUState
*cpu
)
807 #if defined(DEBUG_FLUSH)
808 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
809 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
810 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
811 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
812 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
814 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
815 > tcg_ctx
.code_gen_buffer_size
) {
816 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
818 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
821 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
824 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
827 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
828 /* XXX: flush processor icache at this point if cache flush is
830 tcg_ctx
.tb_ctx
.tb_flush_count
++;
833 #ifdef DEBUG_TB_CHECK
835 static void tb_invalidate_check(target_ulong address
)
837 TranslationBlock
*tb
;
840 address
&= TARGET_PAGE_MASK
;
841 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
842 for (tb
= tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
843 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
844 address
>= tb
->pc
+ tb
->size
)) {
845 printf("ERROR invalidate: address=" TARGET_FMT_lx
846 " PC=%08lx size=%04x\n",
847 address
, (long)tb
->pc
, tb
->size
);
853 /* verify that all the pages have correct rights for code */
854 static void tb_page_check(void)
856 TranslationBlock
*tb
;
857 int i
, flags1
, flags2
;
859 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
860 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
861 tb
= tb
->phys_hash_next
) {
862 flags1
= page_get_flags(tb
->pc
);
863 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
864 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
865 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
866 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
874 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
876 TranslationBlock
*tb1
;
881 *ptb
= tb1
->phys_hash_next
;
884 ptb
= &tb1
->phys_hash_next
;
888 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
890 TranslationBlock
*tb1
;
895 n1
= (uintptr_t)tb1
& 3;
896 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
898 *ptb
= tb1
->page_next
[n1
];
901 ptb
= &tb1
->page_next
[n1
];
905 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
907 TranslationBlock
*tb1
, **ptb
;
910 ptb
= &tb
->jmp_next
[n
];
913 /* find tb(n) in circular list */
916 n1
= (uintptr_t)tb1
& 3;
917 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
918 if (n1
== n
&& tb1
== tb
) {
922 ptb
= &tb1
->jmp_first
;
924 ptb
= &tb1
->jmp_next
[n1
];
927 /* now we can suppress tb(n) from the list */
928 *ptb
= tb
->jmp_next
[n
];
930 tb
->jmp_next
[n
] = NULL
;
934 /* reset the jump entry 'n' of a TB so that it is not chained to
936 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
938 tb_set_jmp_target(tb
, n
, (uintptr_t)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
941 /* invalidate one TB */
942 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
947 tb_page_addr_t phys_pc
;
948 TranslationBlock
*tb1
, *tb2
;
950 /* remove the TB from the hash list */
951 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
952 h
= tb_phys_hash_func(phys_pc
);
953 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
955 /* remove the TB from the page list */
956 if (tb
->page_addr
[0] != page_addr
) {
957 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
958 tb_page_remove(&p
->first_tb
, tb
);
959 invalidate_page_bitmap(p
);
961 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
962 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
963 tb_page_remove(&p
->first_tb
, tb
);
964 invalidate_page_bitmap(p
);
967 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
969 /* remove the TB from the hash list */
970 h
= tb_jmp_cache_hash_func(tb
->pc
);
972 if (cpu
->tb_jmp_cache
[h
] == tb
) {
973 cpu
->tb_jmp_cache
[h
] = NULL
;
977 /* suppress this TB from the two jump lists */
978 tb_jmp_remove(tb
, 0);
979 tb_jmp_remove(tb
, 1);
981 /* suppress any remaining jumps to this TB */
984 n1
= (uintptr_t)tb1
& 3;
988 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
989 tb2
= tb1
->jmp_next
[n1
];
990 tb_reset_jump(tb1
, n1
);
991 tb1
->jmp_next
[n1
] = NULL
;
994 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2); /* fail safe */
996 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
999 static void build_page_bitmap(PageDesc
*p
)
1001 int n
, tb_start
, tb_end
;
1002 TranslationBlock
*tb
;
1004 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1007 while (tb
!= NULL
) {
1008 n
= (uintptr_t)tb
& 3;
1009 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1010 /* NOTE: this is subtle as a TB may span two physical pages */
1012 /* NOTE: tb_end may be after the end of the page, but
1013 it is not a problem */
1014 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1015 tb_end
= tb_start
+ tb
->size
;
1016 if (tb_end
> TARGET_PAGE_SIZE
) {
1017 tb_end
= TARGET_PAGE_SIZE
;
1021 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1023 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1024 tb
= tb
->page_next
[n
];
1028 /* Called with mmap_lock held for user mode emulation. */
1029 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1030 target_ulong pc
, target_ulong cs_base
,
1031 int flags
, int cflags
)
1033 CPUArchState
*env
= cpu
->env_ptr
;
1034 TranslationBlock
*tb
;
1035 tb_page_addr_t phys_pc
, phys_page2
;
1036 target_ulong virt_page2
;
1039 phys_pc
= get_page_addr_code(env
, pc
);
1041 cflags
|= CF_USE_ICOUNT
;
1045 /* flush must be done */
1047 /* cannot fail at this point */
1049 /* Don't forget to invalidate previous TB info. */
1050 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
1052 tb
->tc_ptr
= tcg_ctx
.code_gen_ptr
;
1053 tb
->cs_base
= cs_base
;
1055 tb
->cflags
= cflags
;
1056 cpu_gen_code(env
, tb
, &code_gen_size
);
1057 tcg_ctx
.code_gen_ptr
= (void *)(((uintptr_t)tcg_ctx
.code_gen_ptr
+
1058 code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1060 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1061 /* if we are doing vsyscall don't link the page as it lies in high memory
1062 and tb_alloc_page will abort due to page_l1_map returning NULL */
1063 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1064 && phys_pc
< TARGET_VSYSCALL_END
))
1068 /* check next page if needed */
1069 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1071 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1072 phys_page2
= get_page_addr_code(env
, virt_page2
);
1074 tb_link_page(tb
, phys_pc
, phys_page2
);
1079 * Invalidate all TBs which intersect with the target physical address range
1080 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1081 * 'is_cpu_write_access' should be true if called from a real cpu write
1082 * access: the virtual CPU will exit the current TB if code is modified inside
1085 * Called with mmap_lock held for user-mode emulation
1087 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1089 while (start
< end
) {
1090 tb_invalidate_phys_page_range(start
, end
, 0);
1091 start
&= TARGET_PAGE_MASK
;
1092 start
+= TARGET_PAGE_SIZE
;
1097 * Invalidate all TBs which intersect with the target physical address range
1098 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1099 * 'is_cpu_write_access' should be true if called from a real cpu write
1100 * access: the virtual CPU will exit the current TB if code is modified inside
1103 * Called with mmap_lock held for user-mode emulation
1105 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1106 int is_cpu_write_access
)
1108 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1109 CPUState
*cpu
= current_cpu
;
1110 #if defined(TARGET_HAS_PRECISE_SMC)
1111 CPUArchState
*env
= NULL
;
1113 tb_page_addr_t tb_start
, tb_end
;
1116 #ifdef TARGET_HAS_PRECISE_SMC
1117 int current_tb_not_found
= is_cpu_write_access
;
1118 TranslationBlock
*current_tb
= NULL
;
1119 int current_tb_modified
= 0;
1120 target_ulong current_pc
= 0;
1121 target_ulong current_cs_base
= 0;
1122 int current_flags
= 0;
1123 #endif /* TARGET_HAS_PRECISE_SMC */
1125 p
= page_find(start
>> TARGET_PAGE_BITS
);
1129 #if defined(TARGET_HAS_PRECISE_SMC)
1135 /* we remove all the TBs in the range [start, end[ */
1136 /* XXX: see if in some cases it could be faster to invalidate all
1139 while (tb
!= NULL
) {
1140 n
= (uintptr_t)tb
& 3;
1141 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1142 tb_next
= tb
->page_next
[n
];
1143 /* NOTE: this is subtle as a TB may span two physical pages */
1145 /* NOTE: tb_end may be after the end of the page, but
1146 it is not a problem */
1147 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1148 tb_end
= tb_start
+ tb
->size
;
1150 tb_start
= tb
->page_addr
[1];
1151 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1153 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1154 #ifdef TARGET_HAS_PRECISE_SMC
1155 if (current_tb_not_found
) {
1156 current_tb_not_found
= 0;
1158 if (cpu
->mem_io_pc
) {
1159 /* now we have a real cpu fault */
1160 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1163 if (current_tb
== tb
&&
1164 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1165 /* If we are modifying the current TB, we must stop
1166 its execution. We could be more precise by checking
1167 that the modification is after the current PC, but it
1168 would require a specialized function to partially
1169 restore the CPU state */
1171 current_tb_modified
= 1;
1172 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1173 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1176 #endif /* TARGET_HAS_PRECISE_SMC */
1177 /* we need to do that to handle the case where a signal
1178 occurs while doing tb_phys_invalidate() */
1181 saved_tb
= cpu
->current_tb
;
1182 cpu
->current_tb
= NULL
;
1184 tb_phys_invalidate(tb
, -1);
1186 cpu
->current_tb
= saved_tb
;
1187 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1188 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1194 #if !defined(CONFIG_USER_ONLY)
1195 /* if no code remaining, no need to continue to use slow writes */
1197 invalidate_page_bitmap(p
);
1198 tlb_unprotect_code(start
);
1201 #ifdef TARGET_HAS_PRECISE_SMC
1202 if (current_tb_modified
) {
1203 /* we generate a block containing just the instruction
1204 modifying the memory. It will ensure that it cannot modify
1206 cpu
->current_tb
= NULL
;
1207 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1208 cpu_resume_from_signal(cpu
, NULL
);
1213 /* len must be <= 8 and start must be a multiple of len */
1214 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1220 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1221 cpu_single_env
->mem_io_vaddr
, len
,
1222 cpu_single_env
->eip
,
1223 cpu_single_env
->eip
+
1224 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1227 p
= page_find(start
>> TARGET_PAGE_BITS
);
1231 if (!p
->code_bitmap
&&
1232 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1233 /* build code bitmap */
1234 build_page_bitmap(p
);
1236 if (p
->code_bitmap
) {
1240 nr
= start
& ~TARGET_PAGE_MASK
;
1241 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1242 if (b
& ((1 << len
) - 1)) {
1247 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1251 #if !defined(CONFIG_SOFTMMU)
1252 /* Called with mmap_lock held. */
1253 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1254 uintptr_t pc
, void *puc
,
1257 TranslationBlock
*tb
;
1260 #ifdef TARGET_HAS_PRECISE_SMC
1261 TranslationBlock
*current_tb
= NULL
;
1262 CPUState
*cpu
= current_cpu
;
1263 CPUArchState
*env
= NULL
;
1264 int current_tb_modified
= 0;
1265 target_ulong current_pc
= 0;
1266 target_ulong current_cs_base
= 0;
1267 int current_flags
= 0;
1270 addr
&= TARGET_PAGE_MASK
;
1271 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1276 #ifdef TARGET_HAS_PRECISE_SMC
1277 if (tb
&& pc
!= 0) {
1278 current_tb
= tb_find_pc(pc
);
1284 while (tb
!= NULL
) {
1285 n
= (uintptr_t)tb
& 3;
1286 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1287 #ifdef TARGET_HAS_PRECISE_SMC
1288 if (current_tb
== tb
&&
1289 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1290 /* If we are modifying the current TB, we must stop
1291 its execution. We could be more precise by checking
1292 that the modification is after the current PC, but it
1293 would require a specialized function to partially
1294 restore the CPU state */
1296 current_tb_modified
= 1;
1297 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1298 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1301 #endif /* TARGET_HAS_PRECISE_SMC */
1302 tb_phys_invalidate(tb
, addr
);
1303 tb
= tb
->page_next
[n
];
1306 #ifdef TARGET_HAS_PRECISE_SMC
1307 if (current_tb_modified
) {
1308 /* we generate a block containing just the instruction
1309 modifying the memory. It will ensure that it cannot modify
1311 cpu
->current_tb
= NULL
;
1312 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1316 cpu_resume_from_signal(cpu
, puc
);
1322 /* add the tb in the target page and protect it if necessary
1324 * Called with mmap_lock held for user-mode emulation.
1326 static inline void tb_alloc_page(TranslationBlock
*tb
,
1327 unsigned int n
, tb_page_addr_t page_addr
)
1330 #ifndef CONFIG_USER_ONLY
1331 bool page_already_protected
;
1334 tb
->page_addr
[n
] = page_addr
;
1335 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1336 tb
->page_next
[n
] = p
->first_tb
;
1337 #ifndef CONFIG_USER_ONLY
1338 page_already_protected
= p
->first_tb
!= NULL
;
1340 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1341 invalidate_page_bitmap(p
);
1343 #if defined(CONFIG_USER_ONLY)
1344 if (p
->flags
& PAGE_WRITE
) {
1349 /* force the host page as non writable (writes will have a
1350 page fault + mprotect overhead) */
1351 page_addr
&= qemu_host_page_mask
;
1353 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1354 addr
+= TARGET_PAGE_SIZE
) {
1356 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1361 p2
->flags
&= ~PAGE_WRITE
;
1363 mprotect(g2h(page_addr
), qemu_host_page_size
,
1364 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1365 #ifdef DEBUG_TB_INVALIDATE
1366 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1371 /* if some code is already present, then the pages are already
1372 protected. So we handle the case where only the first TB is
1373 allocated in a physical page */
1374 if (!page_already_protected
) {
1375 tlb_protect_code(page_addr
);
1380 /* add a new TB and link it to the physical page tables. phys_page2 is
1381 * (-1) to indicate that only one page contains the TB.
1383 * Called with mmap_lock held for user-mode emulation.
1385 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1386 tb_page_addr_t phys_page2
)
1389 TranslationBlock
**ptb
;
1391 /* add in the physical hash table */
1392 h
= tb_phys_hash_func(phys_pc
);
1393 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1394 tb
->phys_hash_next
= *ptb
;
1397 /* add in the page list */
1398 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1399 if (phys_page2
!= -1) {
1400 tb_alloc_page(tb
, 1, phys_page2
);
1402 tb
->page_addr
[1] = -1;
1405 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2);
1406 tb
->jmp_next
[0] = NULL
;
1407 tb
->jmp_next
[1] = NULL
;
1409 /* init original jump addresses */
1410 if (tb
->tb_next_offset
[0] != 0xffff) {
1411 tb_reset_jump(tb
, 0);
1413 if (tb
->tb_next_offset
[1] != 0xffff) {
1414 tb_reset_jump(tb
, 1);
1417 #ifdef DEBUG_TB_CHECK
1422 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1423 tb[1].tc_ptr. Return NULL if not found */
1424 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1426 int m_min
, m_max
, m
;
1428 TranslationBlock
*tb
;
1430 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1433 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1434 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1437 /* binary search (cf Knuth) */
1439 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1440 while (m_min
<= m_max
) {
1441 m
= (m_min
+ m_max
) >> 1;
1442 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1443 v
= (uintptr_t)tb
->tc_ptr
;
1446 } else if (tc_ptr
< v
) {
1452 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1455 #if !defined(CONFIG_USER_ONLY)
1456 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1458 ram_addr_t ram_addr
;
1463 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1464 if (!(memory_region_is_ram(mr
)
1465 || memory_region_is_romd(mr
))) {
1469 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1471 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1474 #endif /* !defined(CONFIG_USER_ONLY) */
1476 void tb_check_watchpoint(CPUState
*cpu
)
1478 TranslationBlock
*tb
;
1480 tb
= tb_find_pc(cpu
->mem_io_pc
);
1482 /* We can use retranslation to find the PC. */
1483 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1484 tb_phys_invalidate(tb
, -1);
1486 /* The exception probably happened in a helper. The CPU state should
1487 have been saved before calling it. Fetch the PC from there. */
1488 CPUArchState
*env
= cpu
->env_ptr
;
1489 target_ulong pc
, cs_base
;
1490 tb_page_addr_t addr
;
1493 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1494 addr
= get_page_addr_code(env
, pc
);
1495 tb_invalidate_phys_range(addr
, addr
+ 1);
1499 #ifndef CONFIG_USER_ONLY
1500 /* in deterministic execution mode, instructions doing device I/Os
1501 must be at the end of the TB */
1502 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1504 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1505 CPUArchState
*env
= cpu
->env_ptr
;
1507 TranslationBlock
*tb
;
1509 target_ulong pc
, cs_base
;
1512 tb
= tb_find_pc(retaddr
);
1514 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1517 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1518 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1519 /* Calculate how many instructions had been executed before the fault
1521 n
= n
- cpu
->icount_decr
.u16
.low
;
1522 /* Generate a new TB ending on the I/O insn. */
1524 /* On MIPS and SH, delay slot instructions can only be restarted if
1525 they were already the first instruction in the TB. If this is not
1526 the first instruction in a TB then re-execute the preceding
1528 #if defined(TARGET_MIPS)
1529 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1530 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1531 cpu
->icount_decr
.u16
.low
++;
1532 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1534 #elif defined(TARGET_SH4)
1535 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1538 cpu
->icount_decr
.u16
.low
++;
1539 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1542 /* This should never happen. */
1543 if (n
> CF_COUNT_MASK
) {
1544 cpu_abort(cpu
, "TB too big during recompile");
1547 cflags
= n
| CF_LAST_IO
;
1549 cs_base
= tb
->cs_base
;
1551 tb_phys_invalidate(tb
, -1);
1552 if (tb
->cflags
& CF_NOCACHE
) {
1554 /* Invalidate original TB if this TB was generated in
1555 * cpu_exec_nocache() */
1556 tb_phys_invalidate(tb
->orig_tb
, -1);
1560 /* FIXME: In theory this could raise an exception. In practice
1561 we have already translated the block once so it's probably ok. */
1562 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1563 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1564 the first in the TB) then we end up generating a whole new TB and
1565 repeating the fault, which is horribly inefficient.
1566 Better would be to execute just this insn uncached, or generate a
1568 cpu_resume_from_signal(cpu
, NULL
);
1571 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1575 /* Discard jump cache entries for any tb which might potentially
1576 overlap the flushed page. */
1577 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1578 memset(&cpu
->tb_jmp_cache
[i
], 0,
1579 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1581 i
= tb_jmp_cache_hash_page(addr
);
1582 memset(&cpu
->tb_jmp_cache
[i
], 0,
1583 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1586 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1588 int i
, target_code_size
, max_target_code_size
;
1589 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1590 TranslationBlock
*tb
;
1592 target_code_size
= 0;
1593 max_target_code_size
= 0;
1595 direct_jmp_count
= 0;
1596 direct_jmp2_count
= 0;
1597 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1598 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1599 target_code_size
+= tb
->size
;
1600 if (tb
->size
> max_target_code_size
) {
1601 max_target_code_size
= tb
->size
;
1603 if (tb
->page_addr
[1] != -1) {
1606 if (tb
->tb_next_offset
[0] != 0xffff) {
1608 if (tb
->tb_next_offset
[1] != 0xffff) {
1609 direct_jmp2_count
++;
1613 /* XXX: avoid using doubles ? */
1614 cpu_fprintf(f
, "Translation buffer state:\n");
1615 cpu_fprintf(f
, "gen code size %td/%zd\n",
1616 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1617 tcg_ctx
.code_gen_buffer_max_size
);
1618 cpu_fprintf(f
, "TB count %d/%d\n",
1619 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1620 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1621 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1622 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1623 max_target_code_size
);
1624 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1625 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1626 tcg_ctx
.code_gen_buffer
) /
1627 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1628 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1629 tcg_ctx
.code_gen_buffer
) /
1630 target_code_size
: 0);
1631 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1632 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1633 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1634 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1636 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1637 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1639 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1640 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1641 cpu_fprintf(f
, "\nStatistics:\n");
1642 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1643 cpu_fprintf(f
, "TB invalidate count %d\n",
1644 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1645 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1646 tcg_dump_info(f
, cpu_fprintf
);
1649 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1651 tcg_dump_op_count(f
, cpu_fprintf
);
1654 #else /* CONFIG_USER_ONLY */
1656 void cpu_interrupt(CPUState
*cpu
, int mask
)
1658 cpu
->interrupt_request
|= mask
;
1659 cpu
->tcg_exit_req
= 1;
1663 * Walks guest process memory "regions" one by one
1664 * and calls callback function 'fn' for each region.
1666 struct walk_memory_regions_data
{
1667 walk_memory_regions_fn fn
;
1673 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1674 target_ulong end
, int new_prot
)
1676 if (data
->start
!= -1u) {
1677 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1683 data
->start
= (new_prot
? end
: -1u);
1684 data
->prot
= new_prot
;
1689 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1690 target_ulong base
, int level
, void **lp
)
1696 return walk_memory_regions_end(data
, base
, 0);
1702 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1703 int prot
= pd
[i
].flags
;
1705 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1706 if (prot
!= data
->prot
) {
1707 rc
= walk_memory_regions_end(data
, pa
, prot
);
1716 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1717 pa
= base
| ((target_ulong
)i
<<
1718 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1719 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1729 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1731 struct walk_memory_regions_data data
;
1739 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1740 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1741 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1747 return walk_memory_regions_end(&data
, 0, 0);
1750 static int dump_region(void *priv
, target_ulong start
,
1751 target_ulong end
, abi_ulong prot
)
1753 FILE *f
= (FILE *)priv
;
1755 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1756 " "TARGET_FMT_lx
" %c%c%c\n",
1757 start
, end
, end
- start
,
1758 ((prot
& PAGE_READ
) ? 'r' : '-'),
1759 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1760 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1765 /* dump memory mappings */
1766 void page_dump(FILE *f
)
1768 const int length
= sizeof(target_ulong
) * 2;
1769 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1770 length
, "start", length
, "end", length
, "size", "prot");
1771 walk_memory_regions(f
, dump_region
);
1774 int page_get_flags(target_ulong address
)
1778 p
= page_find(address
>> TARGET_PAGE_BITS
);
1785 /* Modify the flags of a page and invalidate the code if necessary.
1786 The flag PAGE_WRITE_ORG is positioned automatically depending
1787 on PAGE_WRITE. The mmap_lock should already be held. */
1788 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1790 target_ulong addr
, len
;
1792 /* This function should never be called with addresses outside the
1793 guest address space. If this assert fires, it probably indicates
1794 a missing call to h2g_valid. */
1795 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1796 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1798 assert(start
< end
);
1800 start
= start
& TARGET_PAGE_MASK
;
1801 end
= TARGET_PAGE_ALIGN(end
);
1803 if (flags
& PAGE_WRITE
) {
1804 flags
|= PAGE_WRITE_ORG
;
1807 for (addr
= start
, len
= end
- start
;
1809 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1810 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1812 /* If the write protection bit is set, then we invalidate
1814 if (!(p
->flags
& PAGE_WRITE
) &&
1815 (flags
& PAGE_WRITE
) &&
1817 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1823 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1829 /* This function should never be called with addresses outside the
1830 guest address space. If this assert fires, it probably indicates
1831 a missing call to h2g_valid. */
1832 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1833 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1839 if (start
+ len
- 1 < start
) {
1840 /* We've wrapped around. */
1844 /* must do before we loose bits in the next step */
1845 end
= TARGET_PAGE_ALIGN(start
+ len
);
1846 start
= start
& TARGET_PAGE_MASK
;
1848 for (addr
= start
, len
= end
- start
;
1850 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1851 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1855 if (!(p
->flags
& PAGE_VALID
)) {
1859 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1862 if (flags
& PAGE_WRITE
) {
1863 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1866 /* unprotect the page if it was put read-only because it
1867 contains translated code */
1868 if (!(p
->flags
& PAGE_WRITE
)) {
1869 if (!page_unprotect(addr
, 0, NULL
)) {
1878 /* called from signal handler: invalidate the code and unprotect the
1879 page. Return TRUE if the fault was successfully handled. */
1880 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1884 target_ulong host_start
, host_end
, addr
;
1886 /* Technically this isn't safe inside a signal handler. However we
1887 know this only ever happens in a synchronous SEGV handler, so in
1888 practice it seems to be ok. */
1891 p
= page_find(address
>> TARGET_PAGE_BITS
);
1897 /* if the page was really writable, then we change its
1898 protection back to writable */
1899 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
1900 host_start
= address
& qemu_host_page_mask
;
1901 host_end
= host_start
+ qemu_host_page_size
;
1904 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
1905 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1906 p
->flags
|= PAGE_WRITE
;
1909 /* and since the content will be modified, we must invalidate
1910 the corresponding translated code. */
1911 tb_invalidate_phys_page(addr
, pc
, puc
, true);
1912 #ifdef DEBUG_TB_CHECK
1913 tb_invalidate_check(addr
);
1916 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
1925 #endif /* CONFIG_USER_ONLY */