4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
30 #include "qemu-common.h"
31 #define NO_CPU_IO_DEFS
34 #include "disas/disas.h"
36 #if defined(CONFIG_USER_ONLY)
38 #if defined(TARGET_X86_64)
41 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42 #include <sys/param.h>
43 #if __FreeBSD_version >= 700104
44 #define HAVE_KINFO_GETVMMAP
45 #define sigqueue sigqueue_freebsd /* avoid redefinition */
48 #include <machine/profile.h>
57 #include "exec/address-spaces.h"
60 #include "exec/cputlb.h"
61 #include "translate-all.h"
62 #include "qemu/bitmap.h"
63 #include "qemu/timer.h"
65 //#define DEBUG_TB_INVALIDATE
67 /* make various TB consistency checks */
68 //#define DEBUG_TB_CHECK
70 #if !defined(CONFIG_USER_ONLY)
71 /* TB consistency checks only implemented for usermode emulation. */
75 #define SMC_BITMAP_USE_THRESHOLD 10
77 typedef struct PageDesc
{
78 /* list of TBs intersecting this ram page */
79 TranslationBlock
*first_tb
;
80 /* in order to optimize self modifying code, we count the number
81 of lookups we do to a given page to use a bitmap */
82 unsigned int code_write_count
;
83 unsigned long *code_bitmap
;
84 #if defined(CONFIG_USER_ONLY)
89 /* In system mode we want L1_MAP to be based on ram offsets,
90 while in user mode we want it to be based on virtual addresses. */
91 #if !defined(CONFIG_USER_ONLY)
92 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
93 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
95 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
98 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
101 /* Size of the L2 (and L3, etc) page tables. */
103 #define V_L2_SIZE (1 << V_L2_BITS)
105 /* The bits remaining after N lower levels of page tables. */
106 #define V_L1_BITS_REM \
107 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
109 #if V_L1_BITS_REM < 4
110 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
112 #define V_L1_BITS V_L1_BITS_REM
115 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
117 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
119 uintptr_t qemu_real_host_page_size
;
120 uintptr_t qemu_host_page_size
;
121 uintptr_t qemu_host_page_mask
;
123 /* This is a multi-level map on the virtual address space.
124 The bottom level has pointers to PageDesc. */
125 static void *l1_map
[V_L1_SIZE
];
127 /* code generation context */
130 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
131 tb_page_addr_t phys_page2
);
132 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
134 void cpu_gen_init(void)
136 tcg_context_init(&tcg_ctx
);
139 /* return non zero if the very first instruction is invalid so that
140 the virtual CPU can trigger an exception.
142 '*gen_code_size_ptr' contains the size of the generated code (host
145 int cpu_gen_code(CPUArchState
*env
, TranslationBlock
*tb
, int *gen_code_size_ptr
)
147 TCGContext
*s
= &tcg_ctx
;
148 tcg_insn_unit
*gen_code_buf
;
150 #ifdef CONFIG_PROFILER
154 #ifdef CONFIG_PROFILER
155 s
->tb_count1
++; /* includes aborted translations because of
157 ti
= profile_getclock();
161 gen_intermediate_code(env
, tb
);
163 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
165 /* generate machine code */
166 gen_code_buf
= tb
->tc_ptr
;
167 tb
->tb_next_offset
[0] = 0xffff;
168 tb
->tb_next_offset
[1] = 0xffff;
169 s
->tb_next_offset
= tb
->tb_next_offset
;
170 #ifdef USE_DIRECT_JUMP
171 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
174 s
->tb_jmp_offset
= NULL
;
175 s
->tb_next
= tb
->tb_next
;
178 #ifdef CONFIG_PROFILER
180 s
->interm_time
+= profile_getclock() - ti
;
181 s
->code_time
-= profile_getclock();
183 gen_code_size
= tcg_gen_code(s
, gen_code_buf
);
184 *gen_code_size_ptr
= gen_code_size
;
185 #ifdef CONFIG_PROFILER
186 s
->code_time
+= profile_getclock();
187 s
->code_in_len
+= tb
->size
;
188 s
->code_out_len
+= gen_code_size
;
192 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
193 qemu_log("OUT: [size=%d]\n", gen_code_size
);
194 log_disas(tb
->tc_ptr
, gen_code_size
);
202 /* The cpu state corresponding to 'searched_pc' is restored.
204 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
205 uintptr_t searched_pc
)
207 CPUArchState
*env
= cpu
->env_ptr
;
208 TCGContext
*s
= &tcg_ctx
;
211 #ifdef CONFIG_PROFILER
215 #ifdef CONFIG_PROFILER
216 ti
= profile_getclock();
220 gen_intermediate_code_pc(env
, tb
);
222 if (tb
->cflags
& CF_USE_ICOUNT
) {
223 /* Reset the cycle counter to the start of the block. */
224 cpu
->icount_decr
.u16
.low
+= tb
->icount
;
225 /* Clear the IO flag. */
229 /* find opc index corresponding to search_pc */
230 tc_ptr
= (uintptr_t)tb
->tc_ptr
;
231 if (searched_pc
< tc_ptr
)
234 s
->tb_next_offset
= tb
->tb_next_offset
;
235 #ifdef USE_DIRECT_JUMP
236 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
239 s
->tb_jmp_offset
= NULL
;
240 s
->tb_next
= tb
->tb_next
;
242 j
= tcg_gen_code_search_pc(s
, (tcg_insn_unit
*)tc_ptr
,
243 searched_pc
- tc_ptr
);
246 /* now find start of instruction before */
247 while (s
->gen_opc_instr_start
[j
] == 0) {
250 cpu
->icount_decr
.u16
.low
-= s
->gen_opc_icount
[j
];
252 restore_state_to_opc(env
, tb
, j
);
254 #ifdef CONFIG_PROFILER
255 s
->restore_time
+= profile_getclock() - ti
;
261 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
263 TranslationBlock
*tb
;
265 tb
= tb_find_pc(retaddr
);
267 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
268 if (tb
->cflags
& CF_NOCACHE
) {
269 /* one-shot translation, invalidate it immediately */
270 cpu
->current_tb
= NULL
;
271 tb_phys_invalidate(tb
, -1);
280 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
283 VirtualProtect(addr
, size
,
284 PAGE_EXECUTE_READWRITE
, &old_protect
);
287 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
289 unsigned long start
, end
, page_size
;
291 page_size
= getpagesize();
292 start
= (unsigned long)addr
;
293 start
&= ~(page_size
- 1);
295 end
= (unsigned long)addr
+ size
;
296 end
+= page_size
- 1;
297 end
&= ~(page_size
- 1);
299 mprotect((void *)start
, end
- start
,
300 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
304 void page_size_init(void)
306 /* NOTE: we can always suppose that qemu_host_page_size >=
308 qemu_real_host_page_size
= getpagesize();
309 if (qemu_host_page_size
== 0) {
310 qemu_host_page_size
= qemu_real_host_page_size
;
312 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
313 qemu_host_page_size
= TARGET_PAGE_SIZE
;
315 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
318 static void page_init(void)
321 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
323 #ifdef HAVE_KINFO_GETVMMAP
324 struct kinfo_vmentry
*freep
;
327 freep
= kinfo_getvmmap(getpid(), &cnt
);
330 for (i
= 0; i
< cnt
; i
++) {
331 unsigned long startaddr
, endaddr
;
333 startaddr
= freep
[i
].kve_start
;
334 endaddr
= freep
[i
].kve_end
;
335 if (h2g_valid(startaddr
)) {
336 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
338 if (h2g_valid(endaddr
)) {
339 endaddr
= h2g(endaddr
);
340 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
342 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
344 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
355 last_brk
= (unsigned long)sbrk(0);
357 f
= fopen("/compat/linux/proc/self/maps", "r");
362 unsigned long startaddr
, endaddr
;
365 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
367 if (n
== 2 && h2g_valid(startaddr
)) {
368 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
370 if (h2g_valid(endaddr
)) {
371 endaddr
= h2g(endaddr
);
375 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
387 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
393 /* Level 1. Always allocated. */
394 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
397 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
404 p
= g_new0(void *, V_L2_SIZE
);
408 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
416 pd
= g_new0(PageDesc
, V_L2_SIZE
);
420 return pd
+ (index
& (V_L2_SIZE
- 1));
423 static inline PageDesc
*page_find(tb_page_addr_t index
)
425 return page_find_alloc(index
, 0);
428 #if !defined(CONFIG_USER_ONLY)
429 #define mmap_lock() do { } while (0)
430 #define mmap_unlock() do { } while (0)
433 #if defined(CONFIG_USER_ONLY)
434 /* Currently it is not recommended to allocate big chunks of data in
435 user mode. It will change when a dedicated libc will be used. */
436 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
437 region in which the guest needs to run. Revisit this. */
438 #define USE_STATIC_CODE_GEN_BUFFER
441 /* ??? Should configure for this, not list operating systems here. */
442 #if (defined(__linux__) \
443 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
444 || defined(__DragonFly__) || defined(__OpenBSD__) \
445 || defined(__NetBSD__))
449 /* Minimum size of the code gen buffer. This number is randomly chosen,
450 but not so small that we can't have a fair number of TB's live. */
451 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
453 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
454 indicated, this is constrained by the range of direct branches on the
455 host cpu, as used by the TCG implementation of goto_tb. */
456 #if defined(__x86_64__)
457 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
458 #elif defined(__sparc__)
459 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
460 #elif defined(__aarch64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
462 #elif defined(__arm__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
464 #elif defined(__s390x__)
465 /* We have a +- 4GB range on the branches; leave some slop. */
466 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
467 #elif defined(__mips__)
468 /* We have a 256MB branch region, but leave room to make sure the
469 main executable is also within that region. */
470 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
472 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
475 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
477 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
478 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
479 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
481 static inline size_t size_code_gen_buffer(size_t tb_size
)
483 /* Size the buffer. */
485 #ifdef USE_STATIC_CODE_GEN_BUFFER
486 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
488 /* ??? Needs adjustments. */
489 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
490 static buffer, we could size this on RESERVED_VA, on the text
491 segment size of the executable, or continue to use the default. */
492 tb_size
= (unsigned long)(ram_size
/ 4);
495 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
496 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
498 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
499 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
501 tcg_ctx
.code_gen_buffer_size
= tb_size
;
506 /* In order to use J and JAL within the code_gen_buffer, we require
507 that the buffer not cross a 256MB boundary. */
508 static inline bool cross_256mb(void *addr
, size_t size
)
510 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & 0xf0000000;
513 /* We weren't able to allocate a buffer without crossing that boundary,
514 so make do with the larger portion of the buffer that doesn't cross.
515 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
516 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
518 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & 0xf0000000);
519 size_t size2
= buf1
+ size1
- buf2
;
527 tcg_ctx
.code_gen_buffer_size
= size1
;
532 #ifdef USE_STATIC_CODE_GEN_BUFFER
533 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
534 __attribute__((aligned(CODE_GEN_ALIGN
)));
536 static inline void *alloc_code_gen_buffer(void)
538 void *buf
= static_code_gen_buffer
;
540 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
541 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
544 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
547 #elif defined(USE_MMAP)
548 static inline void *alloc_code_gen_buffer(void)
550 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
554 /* Constrain the position of the buffer based on the host cpu.
555 Note that these addresses are chosen in concert with the
556 addresses assigned in the relevant linker script file. */
557 # if defined(__PIE__) || defined(__PIC__)
558 /* Don't bother setting a preferred location if we're building
559 a position-independent executable. We're more likely to get
560 an address near the main executable if we let the kernel
561 choose the address. */
562 # elif defined(__x86_64__) && defined(MAP_32BIT)
563 /* Force the memory down into low memory with the executable.
564 Leave the choice of exact location with the kernel. */
566 /* Cannot expect to map more than 800MB in low memory. */
567 if (tcg_ctx
.code_gen_buffer_size
> 800u * 1024 * 1024) {
568 tcg_ctx
.code_gen_buffer_size
= 800u * 1024 * 1024;
570 # elif defined(__sparc__)
571 start
= 0x40000000ul
;
572 # elif defined(__s390x__)
573 start
= 0x90000000ul
;
574 # elif defined(__mips__)
575 /* ??? We ought to more explicitly manage layout for softmmu too. */
576 # ifdef CONFIG_USER_ONLY
577 start
= 0x68000000ul
;
578 # elif _MIPS_SIM == _ABI64
579 start
= 0x128000000ul
;
581 start
= 0x08000000ul
;
585 buf
= mmap((void *)start
, tcg_ctx
.code_gen_buffer_size
,
586 PROT_WRITE
| PROT_READ
| PROT_EXEC
, flags
, -1, 0);
587 if (buf
== MAP_FAILED
) {
592 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
593 /* Try again, with the original still mapped, to avoid re-acquiring
594 that 256mb crossing. This time don't specify an address. */
595 size_t size2
, size1
= tcg_ctx
.code_gen_buffer_size
;
596 void *buf2
= mmap(NULL
, size1
, PROT_WRITE
| PROT_READ
| PROT_EXEC
,
598 if (buf2
!= MAP_FAILED
) {
599 if (!cross_256mb(buf2
, size1
)) {
600 /* Success! Use the new buffer. */
604 /* Failure. Work with what we had. */
608 /* Split the original buffer. Free the smaller half. */
609 buf2
= split_cross_256mb(buf
, size1
);
610 size2
= tcg_ctx
.code_gen_buffer_size
;
611 munmap(buf
+ (buf
== buf2
? size2
: 0), size1
- size2
);
619 static inline void *alloc_code_gen_buffer(void)
621 void *buf
= g_try_malloc(tcg_ctx
.code_gen_buffer_size
);
628 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
629 void *buf2
= g_malloc(tcg_ctx
.code_gen_buffer_size
);
630 if (buf2
!= NULL
&& !cross_256mb(buf2
, size1
)) {
631 /* Success! Use the new buffer. */
635 /* Failure. Work with what we had. Since this is malloc
636 and not mmap, we can't free the other half. */
638 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
643 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
646 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
648 static inline void code_gen_alloc(size_t tb_size
)
650 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
651 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
652 if (tcg_ctx
.code_gen_buffer
== NULL
) {
653 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
657 qemu_madvise(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
,
660 /* Steal room for the prologue at the end of the buffer. This ensures
661 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
662 from TB's to the prologue are going to be in range. It also means
663 that we don't need to mark (additional) portions of the data segment
665 tcg_ctx
.code_gen_prologue
= tcg_ctx
.code_gen_buffer
+
666 tcg_ctx
.code_gen_buffer_size
- 1024;
667 tcg_ctx
.code_gen_buffer_size
-= 1024;
669 tcg_ctx
.code_gen_buffer_max_size
= tcg_ctx
.code_gen_buffer_size
-
670 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
671 tcg_ctx
.code_gen_max_blocks
= tcg_ctx
.code_gen_buffer_size
/
672 CODE_GEN_AVG_BLOCK_SIZE
;
674 g_malloc(tcg_ctx
.code_gen_max_blocks
* sizeof(TranslationBlock
));
677 /* Must be called before using the QEMU cpus. 'tb_size' is the size
678 (in bytes) allocated to the translation buffer. Zero means default
680 void tcg_exec_init(uintptr_t tb_size
)
683 code_gen_alloc(tb_size
);
684 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
685 tcg_register_jit(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
);
687 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
688 /* There's no guest base to take into account, so go ahead and
689 initialize the prologue now. */
690 tcg_prologue_init(&tcg_ctx
);
694 bool tcg_enabled(void)
696 return tcg_ctx
.code_gen_buffer
!= NULL
;
699 /* Allocate a new translation block. Flush the translation buffer if
700 too many translation blocks or too much generated code. */
701 static TranslationBlock
*tb_alloc(target_ulong pc
)
703 TranslationBlock
*tb
;
705 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
||
706 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) >=
707 tcg_ctx
.code_gen_buffer_max_size
) {
710 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
716 void tb_free(TranslationBlock
*tb
)
718 /* In practice this is mostly used for single use temporary TB
719 Ignore the hard cases and just back up if this TB happens to
720 be the last one generated. */
721 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
722 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
723 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
724 tcg_ctx
.tb_ctx
.nb_tbs
--;
728 static inline void invalidate_page_bitmap(PageDesc
*p
)
730 if (p
->code_bitmap
) {
731 g_free(p
->code_bitmap
);
732 p
->code_bitmap
= NULL
;
734 p
->code_write_count
= 0;
737 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
738 static void page_flush_tb_1(int level
, void **lp
)
748 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
749 pd
[i
].first_tb
= NULL
;
750 invalidate_page_bitmap(pd
+ i
);
755 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
756 page_flush_tb_1(level
- 1, pp
+ i
);
761 static void page_flush_tb(void)
765 for (i
= 0; i
< V_L1_SIZE
; i
++) {
766 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
770 /* flush all the translation blocks */
771 /* XXX: tb_flush is currently not thread safe */
772 void tb_flush(CPUArchState
*env1
)
774 CPUState
*cpu
= ENV_GET_CPU(env1
);
776 #if defined(DEBUG_FLUSH)
777 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
778 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
779 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
780 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
781 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
783 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
784 > tcg_ctx
.code_gen_buffer_size
) {
785 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
787 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
790 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
793 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
796 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
797 /* XXX: flush processor icache at this point if cache flush is
799 tcg_ctx
.tb_ctx
.tb_flush_count
++;
802 #ifdef DEBUG_TB_CHECK
804 static void tb_invalidate_check(target_ulong address
)
806 TranslationBlock
*tb
;
809 address
&= TARGET_PAGE_MASK
;
810 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
811 for (tb
= tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
812 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
813 address
>= tb
->pc
+ tb
->size
)) {
814 printf("ERROR invalidate: address=" TARGET_FMT_lx
815 " PC=%08lx size=%04x\n",
816 address
, (long)tb
->pc
, tb
->size
);
822 /* verify that all the pages have correct rights for code */
823 static void tb_page_check(void)
825 TranslationBlock
*tb
;
826 int i
, flags1
, flags2
;
828 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
829 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
830 tb
= tb
->phys_hash_next
) {
831 flags1
= page_get_flags(tb
->pc
);
832 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
833 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
834 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
835 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
843 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
845 TranslationBlock
*tb1
;
850 *ptb
= tb1
->phys_hash_next
;
853 ptb
= &tb1
->phys_hash_next
;
857 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
859 TranslationBlock
*tb1
;
864 n1
= (uintptr_t)tb1
& 3;
865 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
867 *ptb
= tb1
->page_next
[n1
];
870 ptb
= &tb1
->page_next
[n1
];
874 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
876 TranslationBlock
*tb1
, **ptb
;
879 ptb
= &tb
->jmp_next
[n
];
882 /* find tb(n) in circular list */
885 n1
= (uintptr_t)tb1
& 3;
886 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
887 if (n1
== n
&& tb1
== tb
) {
891 ptb
= &tb1
->jmp_first
;
893 ptb
= &tb1
->jmp_next
[n1
];
896 /* now we can suppress tb(n) from the list */
897 *ptb
= tb
->jmp_next
[n
];
899 tb
->jmp_next
[n
] = NULL
;
903 /* reset the jump entry 'n' of a TB so that it is not chained to
905 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
907 tb_set_jmp_target(tb
, n
, (uintptr_t)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
910 /* invalidate one TB */
911 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
916 tb_page_addr_t phys_pc
;
917 TranslationBlock
*tb1
, *tb2
;
919 /* remove the TB from the hash list */
920 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
921 h
= tb_phys_hash_func(phys_pc
);
922 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
924 /* remove the TB from the page list */
925 if (tb
->page_addr
[0] != page_addr
) {
926 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
927 tb_page_remove(&p
->first_tb
, tb
);
928 invalidate_page_bitmap(p
);
930 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
931 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
932 tb_page_remove(&p
->first_tb
, tb
);
933 invalidate_page_bitmap(p
);
936 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
938 /* remove the TB from the hash list */
939 h
= tb_jmp_cache_hash_func(tb
->pc
);
941 if (cpu
->tb_jmp_cache
[h
] == tb
) {
942 cpu
->tb_jmp_cache
[h
] = NULL
;
946 /* suppress this TB from the two jump lists */
947 tb_jmp_remove(tb
, 0);
948 tb_jmp_remove(tb
, 1);
950 /* suppress any remaining jumps to this TB */
953 n1
= (uintptr_t)tb1
& 3;
957 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
958 tb2
= tb1
->jmp_next
[n1
];
959 tb_reset_jump(tb1
, n1
);
960 tb1
->jmp_next
[n1
] = NULL
;
963 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2); /* fail safe */
965 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
968 static void build_page_bitmap(PageDesc
*p
)
970 int n
, tb_start
, tb_end
;
971 TranslationBlock
*tb
;
973 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
977 n
= (uintptr_t)tb
& 3;
978 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
979 /* NOTE: this is subtle as a TB may span two physical pages */
981 /* NOTE: tb_end may be after the end of the page, but
982 it is not a problem */
983 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
984 tb_end
= tb_start
+ tb
->size
;
985 if (tb_end
> TARGET_PAGE_SIZE
) {
986 tb_end
= TARGET_PAGE_SIZE
;
990 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
992 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
993 tb
= tb
->page_next
[n
];
997 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
998 target_ulong pc
, target_ulong cs_base
,
999 int flags
, int cflags
)
1001 CPUArchState
*env
= cpu
->env_ptr
;
1002 TranslationBlock
*tb
;
1003 tb_page_addr_t phys_pc
, phys_page2
;
1004 target_ulong virt_page2
;
1007 phys_pc
= get_page_addr_code(env
, pc
);
1009 cflags
|= CF_USE_ICOUNT
;
1013 /* flush must be done */
1015 /* cannot fail at this point */
1017 /* Don't forget to invalidate previous TB info. */
1018 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
1020 tb
->tc_ptr
= tcg_ctx
.code_gen_ptr
;
1021 tb
->cs_base
= cs_base
;
1023 tb
->cflags
= cflags
;
1024 cpu_gen_code(env
, tb
, &code_gen_size
);
1025 tcg_ctx
.code_gen_ptr
= (void *)(((uintptr_t)tcg_ctx
.code_gen_ptr
+
1026 code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1028 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1029 /* if we are doing vsyscall don't link the page as it lies in high memory
1030 and tb_alloc_page will abort due to page_l1_map returning NULL */
1031 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1032 && phys_pc
< TARGET_VSYSCALL_END
))
1036 /* check next page if needed */
1037 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1039 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1040 phys_page2
= get_page_addr_code(env
, virt_page2
);
1042 tb_link_page(tb
, phys_pc
, phys_page2
);
1047 * Invalidate all TBs which intersect with the target physical address range
1048 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1049 * 'is_cpu_write_access' should be true if called from a real cpu write
1050 * access: the virtual CPU will exit the current TB if code is modified inside
1053 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
,
1054 int is_cpu_write_access
)
1056 while (start
< end
) {
1057 tb_invalidate_phys_page_range(start
, end
, is_cpu_write_access
);
1058 start
&= TARGET_PAGE_MASK
;
1059 start
+= TARGET_PAGE_SIZE
;
1064 * Invalidate all TBs which intersect with the target physical address range
1065 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1066 * 'is_cpu_write_access' should be true if called from a real cpu write
1067 * access: the virtual CPU will exit the current TB if code is modified inside
1070 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1071 int is_cpu_write_access
)
1073 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1074 CPUState
*cpu
= current_cpu
;
1075 #if defined(TARGET_HAS_PRECISE_SMC)
1076 CPUArchState
*env
= NULL
;
1078 tb_page_addr_t tb_start
, tb_end
;
1081 #ifdef TARGET_HAS_PRECISE_SMC
1082 int current_tb_not_found
= is_cpu_write_access
;
1083 TranslationBlock
*current_tb
= NULL
;
1084 int current_tb_modified
= 0;
1085 target_ulong current_pc
= 0;
1086 target_ulong current_cs_base
= 0;
1087 int current_flags
= 0;
1088 #endif /* TARGET_HAS_PRECISE_SMC */
1090 p
= page_find(start
>> TARGET_PAGE_BITS
);
1094 if (!p
->code_bitmap
&&
1095 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1096 is_cpu_write_access
) {
1097 /* build code bitmap */
1098 build_page_bitmap(p
);
1100 #if defined(TARGET_HAS_PRECISE_SMC)
1106 /* we remove all the TBs in the range [start, end[ */
1107 /* XXX: see if in some cases it could be faster to invalidate all
1110 while (tb
!= NULL
) {
1111 n
= (uintptr_t)tb
& 3;
1112 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1113 tb_next
= tb
->page_next
[n
];
1114 /* NOTE: this is subtle as a TB may span two physical pages */
1116 /* NOTE: tb_end may be after the end of the page, but
1117 it is not a problem */
1118 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1119 tb_end
= tb_start
+ tb
->size
;
1121 tb_start
= tb
->page_addr
[1];
1122 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1124 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1125 #ifdef TARGET_HAS_PRECISE_SMC
1126 if (current_tb_not_found
) {
1127 current_tb_not_found
= 0;
1129 if (cpu
->mem_io_pc
) {
1130 /* now we have a real cpu fault */
1131 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1134 if (current_tb
== tb
&&
1135 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1136 /* If we are modifying the current TB, we must stop
1137 its execution. We could be more precise by checking
1138 that the modification is after the current PC, but it
1139 would require a specialized function to partially
1140 restore the CPU state */
1142 current_tb_modified
= 1;
1143 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1144 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1147 #endif /* TARGET_HAS_PRECISE_SMC */
1148 /* we need to do that to handle the case where a signal
1149 occurs while doing tb_phys_invalidate() */
1152 saved_tb
= cpu
->current_tb
;
1153 cpu
->current_tb
= NULL
;
1155 tb_phys_invalidate(tb
, -1);
1157 cpu
->current_tb
= saved_tb
;
1158 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1159 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1165 #if !defined(CONFIG_USER_ONLY)
1166 /* if no code remaining, no need to continue to use slow writes */
1168 invalidate_page_bitmap(p
);
1169 if (is_cpu_write_access
) {
1170 tlb_unprotect_code_phys(cpu
, start
, cpu
->mem_io_vaddr
);
1174 #ifdef TARGET_HAS_PRECISE_SMC
1175 if (current_tb_modified
) {
1176 /* we generate a block containing just the instruction
1177 modifying the memory. It will ensure that it cannot modify
1179 cpu
->current_tb
= NULL
;
1180 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1181 cpu_resume_from_signal(cpu
, NULL
);
1186 /* len must be <= 8 and start must be a multiple of len */
1187 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1193 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1194 cpu_single_env
->mem_io_vaddr
, len
,
1195 cpu_single_env
->eip
,
1196 cpu_single_env
->eip
+
1197 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1200 p
= page_find(start
>> TARGET_PAGE_BITS
);
1204 if (p
->code_bitmap
) {
1208 nr
= start
& ~TARGET_PAGE_MASK
;
1209 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1210 if (b
& ((1 << len
) - 1)) {
1215 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1219 #if !defined(CONFIG_SOFTMMU)
1220 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1221 uintptr_t pc
, void *puc
,
1224 TranslationBlock
*tb
;
1227 #ifdef TARGET_HAS_PRECISE_SMC
1228 TranslationBlock
*current_tb
= NULL
;
1229 CPUState
*cpu
= current_cpu
;
1230 CPUArchState
*env
= NULL
;
1231 int current_tb_modified
= 0;
1232 target_ulong current_pc
= 0;
1233 target_ulong current_cs_base
= 0;
1234 int current_flags
= 0;
1237 addr
&= TARGET_PAGE_MASK
;
1238 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1243 #ifdef TARGET_HAS_PRECISE_SMC
1244 if (tb
&& pc
!= 0) {
1245 current_tb
= tb_find_pc(pc
);
1251 while (tb
!= NULL
) {
1252 n
= (uintptr_t)tb
& 3;
1253 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1254 #ifdef TARGET_HAS_PRECISE_SMC
1255 if (current_tb
== tb
&&
1256 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1257 /* If we are modifying the current TB, we must stop
1258 its execution. We could be more precise by checking
1259 that the modification is after the current PC, but it
1260 would require a specialized function to partially
1261 restore the CPU state */
1263 current_tb_modified
= 1;
1264 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1265 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1268 #endif /* TARGET_HAS_PRECISE_SMC */
1269 tb_phys_invalidate(tb
, addr
);
1270 tb
= tb
->page_next
[n
];
1273 #ifdef TARGET_HAS_PRECISE_SMC
1274 if (current_tb_modified
) {
1275 /* we generate a block containing just the instruction
1276 modifying the memory. It will ensure that it cannot modify
1278 cpu
->current_tb
= NULL
;
1279 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1283 cpu_resume_from_signal(cpu
, puc
);
1289 /* add the tb in the target page and protect it if necessary */
1290 static inline void tb_alloc_page(TranslationBlock
*tb
,
1291 unsigned int n
, tb_page_addr_t page_addr
)
1294 #ifndef CONFIG_USER_ONLY
1295 bool page_already_protected
;
1298 tb
->page_addr
[n
] = page_addr
;
1299 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1300 tb
->page_next
[n
] = p
->first_tb
;
1301 #ifndef CONFIG_USER_ONLY
1302 page_already_protected
= p
->first_tb
!= NULL
;
1304 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1305 invalidate_page_bitmap(p
);
1307 #if defined(CONFIG_USER_ONLY)
1308 if (p
->flags
& PAGE_WRITE
) {
1313 /* force the host page as non writable (writes will have a
1314 page fault + mprotect overhead) */
1315 page_addr
&= qemu_host_page_mask
;
1317 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1318 addr
+= TARGET_PAGE_SIZE
) {
1320 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1325 p2
->flags
&= ~PAGE_WRITE
;
1327 mprotect(g2h(page_addr
), qemu_host_page_size
,
1328 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1329 #ifdef DEBUG_TB_INVALIDATE
1330 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1335 /* if some code is already present, then the pages are already
1336 protected. So we handle the case where only the first TB is
1337 allocated in a physical page */
1338 if (!page_already_protected
) {
1339 tlb_protect_code(page_addr
);
1344 /* add a new TB and link it to the physical page tables. phys_page2 is
1345 (-1) to indicate that only one page contains the TB. */
1346 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1347 tb_page_addr_t phys_page2
)
1350 TranslationBlock
**ptb
;
1352 /* Grab the mmap lock to stop another thread invalidating this TB
1353 before we are done. */
1355 /* add in the physical hash table */
1356 h
= tb_phys_hash_func(phys_pc
);
1357 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1358 tb
->phys_hash_next
= *ptb
;
1361 /* add in the page list */
1362 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1363 if (phys_page2
!= -1) {
1364 tb_alloc_page(tb
, 1, phys_page2
);
1366 tb
->page_addr
[1] = -1;
1369 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2);
1370 tb
->jmp_next
[0] = NULL
;
1371 tb
->jmp_next
[1] = NULL
;
1373 /* init original jump addresses */
1374 if (tb
->tb_next_offset
[0] != 0xffff) {
1375 tb_reset_jump(tb
, 0);
1377 if (tb
->tb_next_offset
[1] != 0xffff) {
1378 tb_reset_jump(tb
, 1);
1381 #ifdef DEBUG_TB_CHECK
1387 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1388 tb[1].tc_ptr. Return NULL if not found */
1389 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1391 int m_min
, m_max
, m
;
1393 TranslationBlock
*tb
;
1395 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1398 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1399 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1402 /* binary search (cf Knuth) */
1404 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1405 while (m_min
<= m_max
) {
1406 m
= (m_min
+ m_max
) >> 1;
1407 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1408 v
= (uintptr_t)tb
->tc_ptr
;
1411 } else if (tc_ptr
< v
) {
1417 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1420 #if !defined(CONFIG_USER_ONLY)
1421 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1423 ram_addr_t ram_addr
;
1428 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1429 if (!(memory_region_is_ram(mr
)
1430 || memory_region_is_romd(mr
))) {
1434 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1436 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1439 #endif /* !defined(CONFIG_USER_ONLY) */
1441 void tb_check_watchpoint(CPUState
*cpu
)
1443 TranslationBlock
*tb
;
1445 tb
= tb_find_pc(cpu
->mem_io_pc
);
1447 cpu_abort(cpu
, "check_watchpoint: could not find TB for pc=%p",
1448 (void *)cpu
->mem_io_pc
);
1450 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1451 tb_phys_invalidate(tb
, -1);
1454 #ifndef CONFIG_USER_ONLY
1455 /* mask must never be zero, except for A20 change call */
1456 static void tcg_handle_interrupt(CPUState
*cpu
, int mask
)
1460 old_mask
= cpu
->interrupt_request
;
1461 cpu
->interrupt_request
|= mask
;
1464 * If called from iothread context, wake the target cpu in
1467 if (!qemu_cpu_is_self(cpu
)) {
1473 cpu
->icount_decr
.u16
.high
= 0xffff;
1474 if (!cpu_can_do_io(cpu
)
1475 && (mask
& ~old_mask
) != 0) {
1476 cpu_abort(cpu
, "Raised interrupt while not in I/O function");
1479 cpu
->tcg_exit_req
= 1;
1483 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1485 /* in deterministic execution mode, instructions doing device I/Os
1486 must be at the end of the TB */
1487 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1489 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1490 CPUArchState
*env
= cpu
->env_ptr
;
1492 TranslationBlock
*tb
;
1494 target_ulong pc
, cs_base
;
1497 tb
= tb_find_pc(retaddr
);
1499 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1502 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1503 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1504 /* Calculate how many instructions had been executed before the fault
1506 n
= n
- cpu
->icount_decr
.u16
.low
;
1507 /* Generate a new TB ending on the I/O insn. */
1509 /* On MIPS and SH, delay slot instructions can only be restarted if
1510 they were already the first instruction in the TB. If this is not
1511 the first instruction in a TB then re-execute the preceding
1513 #if defined(TARGET_MIPS)
1514 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1515 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1516 cpu
->icount_decr
.u16
.low
++;
1517 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1519 #elif defined(TARGET_SH4)
1520 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1523 cpu
->icount_decr
.u16
.low
++;
1524 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1527 /* This should never happen. */
1528 if (n
> CF_COUNT_MASK
) {
1529 cpu_abort(cpu
, "TB too big during recompile");
1532 cflags
= n
| CF_LAST_IO
;
1534 cs_base
= tb
->cs_base
;
1536 tb_phys_invalidate(tb
, -1);
1537 /* FIXME: In theory this could raise an exception. In practice
1538 we have already translated the block once so it's probably ok. */
1539 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1540 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1541 the first in the TB) then we end up generating a whole new TB and
1542 repeating the fault, which is horribly inefficient.
1543 Better would be to execute just this insn uncached, or generate a
1545 cpu_resume_from_signal(cpu
, NULL
);
1548 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1552 /* Discard jump cache entries for any tb which might potentially
1553 overlap the flushed page. */
1554 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1555 memset(&cpu
->tb_jmp_cache
[i
], 0,
1556 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1558 i
= tb_jmp_cache_hash_page(addr
);
1559 memset(&cpu
->tb_jmp_cache
[i
], 0,
1560 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1563 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1565 int i
, target_code_size
, max_target_code_size
;
1566 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1567 TranslationBlock
*tb
;
1569 target_code_size
= 0;
1570 max_target_code_size
= 0;
1572 direct_jmp_count
= 0;
1573 direct_jmp2_count
= 0;
1574 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1575 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1576 target_code_size
+= tb
->size
;
1577 if (tb
->size
> max_target_code_size
) {
1578 max_target_code_size
= tb
->size
;
1580 if (tb
->page_addr
[1] != -1) {
1583 if (tb
->tb_next_offset
[0] != 0xffff) {
1585 if (tb
->tb_next_offset
[1] != 0xffff) {
1586 direct_jmp2_count
++;
1590 /* XXX: avoid using doubles ? */
1591 cpu_fprintf(f
, "Translation buffer state:\n");
1592 cpu_fprintf(f
, "gen code size %td/%zd\n",
1593 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1594 tcg_ctx
.code_gen_buffer_max_size
);
1595 cpu_fprintf(f
, "TB count %d/%d\n",
1596 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1597 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1598 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1599 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1600 max_target_code_size
);
1601 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1602 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1603 tcg_ctx
.code_gen_buffer
) /
1604 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1605 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1606 tcg_ctx
.code_gen_buffer
) /
1607 target_code_size
: 0);
1608 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1609 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1610 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1611 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1613 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1614 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1616 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1617 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1618 cpu_fprintf(f
, "\nStatistics:\n");
1619 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1620 cpu_fprintf(f
, "TB invalidate count %d\n",
1621 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1622 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1623 tcg_dump_info(f
, cpu_fprintf
);
1626 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1628 tcg_dump_op_count(f
, cpu_fprintf
);
1631 #else /* CONFIG_USER_ONLY */
1633 void cpu_interrupt(CPUState
*cpu
, int mask
)
1635 cpu
->interrupt_request
|= mask
;
1636 cpu
->tcg_exit_req
= 1;
1640 * Walks guest process memory "regions" one by one
1641 * and calls callback function 'fn' for each region.
1643 struct walk_memory_regions_data
{
1644 walk_memory_regions_fn fn
;
1650 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1651 target_ulong end
, int new_prot
)
1653 if (data
->start
!= -1u) {
1654 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1660 data
->start
= (new_prot
? end
: -1u);
1661 data
->prot
= new_prot
;
1666 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1667 target_ulong base
, int level
, void **lp
)
1673 return walk_memory_regions_end(data
, base
, 0);
1679 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1680 int prot
= pd
[i
].flags
;
1682 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1683 if (prot
!= data
->prot
) {
1684 rc
= walk_memory_regions_end(data
, pa
, prot
);
1693 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1694 pa
= base
| ((target_ulong
)i
<<
1695 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1696 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1706 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1708 struct walk_memory_regions_data data
;
1716 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1717 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1718 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1724 return walk_memory_regions_end(&data
, 0, 0);
1727 static int dump_region(void *priv
, target_ulong start
,
1728 target_ulong end
, abi_ulong prot
)
1730 FILE *f
= (FILE *)priv
;
1732 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1733 " "TARGET_FMT_lx
" %c%c%c\n",
1734 start
, end
, end
- start
,
1735 ((prot
& PAGE_READ
) ? 'r' : '-'),
1736 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1737 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1742 /* dump memory mappings */
1743 void page_dump(FILE *f
)
1745 const int length
= sizeof(target_ulong
) * 2;
1746 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1747 length
, "start", length
, "end", length
, "size", "prot");
1748 walk_memory_regions(f
, dump_region
);
1751 int page_get_flags(target_ulong address
)
1755 p
= page_find(address
>> TARGET_PAGE_BITS
);
1762 /* Modify the flags of a page and invalidate the code if necessary.
1763 The flag PAGE_WRITE_ORG is positioned automatically depending
1764 on PAGE_WRITE. The mmap_lock should already be held. */
1765 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1767 target_ulong addr
, len
;
1769 /* This function should never be called with addresses outside the
1770 guest address space. If this assert fires, it probably indicates
1771 a missing call to h2g_valid. */
1772 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1773 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1775 assert(start
< end
);
1777 start
= start
& TARGET_PAGE_MASK
;
1778 end
= TARGET_PAGE_ALIGN(end
);
1780 if (flags
& PAGE_WRITE
) {
1781 flags
|= PAGE_WRITE_ORG
;
1784 for (addr
= start
, len
= end
- start
;
1786 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1787 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1789 /* If the write protection bit is set, then we invalidate
1791 if (!(p
->flags
& PAGE_WRITE
) &&
1792 (flags
& PAGE_WRITE
) &&
1794 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1800 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1806 /* This function should never be called with addresses outside the
1807 guest address space. If this assert fires, it probably indicates
1808 a missing call to h2g_valid. */
1809 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1810 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1816 if (start
+ len
- 1 < start
) {
1817 /* We've wrapped around. */
1821 /* must do before we loose bits in the next step */
1822 end
= TARGET_PAGE_ALIGN(start
+ len
);
1823 start
= start
& TARGET_PAGE_MASK
;
1825 for (addr
= start
, len
= end
- start
;
1827 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1828 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1832 if (!(p
->flags
& PAGE_VALID
)) {
1836 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1839 if (flags
& PAGE_WRITE
) {
1840 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1843 /* unprotect the page if it was put read-only because it
1844 contains translated code */
1845 if (!(p
->flags
& PAGE_WRITE
)) {
1846 if (!page_unprotect(addr
, 0, NULL
)) {
1855 /* called from signal handler: invalidate the code and unprotect the
1856 page. Return TRUE if the fault was successfully handled. */
1857 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1861 target_ulong host_start
, host_end
, addr
;
1863 /* Technically this isn't safe inside a signal handler. However we
1864 know this only ever happens in a synchronous SEGV handler, so in
1865 practice it seems to be ok. */
1868 p
= page_find(address
>> TARGET_PAGE_BITS
);
1874 /* if the page was really writable, then we change its
1875 protection back to writable */
1876 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
1877 host_start
= address
& qemu_host_page_mask
;
1878 host_end
= host_start
+ qemu_host_page_size
;
1881 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
1882 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1883 p
->flags
|= PAGE_WRITE
;
1886 /* and since the content will be modified, we must invalidate
1887 the corresponding translated code. */
1888 tb_invalidate_phys_page(addr
, pc
, puc
, true);
1889 #ifdef DEBUG_TB_CHECK
1890 tb_invalidate_check(addr
);
1893 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
1902 #endif /* CONFIG_USER_ONLY */