4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
30 #include "qemu-common.h"
31 #define NO_CPU_IO_DEFS
34 #include "disas/disas.h"
36 #if defined(CONFIG_USER_ONLY)
38 #if defined(TARGET_X86_64)
41 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42 #include <sys/param.h>
43 #if __FreeBSD_version >= 700104
44 #define HAVE_KINFO_GETVMMAP
45 #define sigqueue sigqueue_freebsd /* avoid redefinition */
48 #include <machine/profile.h>
57 #include "exec/address-spaces.h"
60 #include "exec/cputlb.h"
61 #include "exec/tb-hash.h"
62 #include "translate-all.h"
63 #include "qemu/bitmap.h"
64 #include "qemu/timer.h"
66 //#define DEBUG_TB_INVALIDATE
68 /* make various TB consistency checks */
69 //#define DEBUG_TB_CHECK
71 #if !defined(CONFIG_USER_ONLY)
72 /* TB consistency checks only implemented for usermode emulation. */
76 #define SMC_BITMAP_USE_THRESHOLD 10
78 typedef struct PageDesc
{
79 /* list of TBs intersecting this ram page */
80 TranslationBlock
*first_tb
;
81 /* in order to optimize self modifying code, we count the number
82 of lookups we do to a given page to use a bitmap */
83 unsigned int code_write_count
;
84 unsigned long *code_bitmap
;
85 #if defined(CONFIG_USER_ONLY)
90 /* In system mode we want L1_MAP to be based on ram offsets,
91 while in user mode we want it to be based on virtual addresses. */
92 #if !defined(CONFIG_USER_ONLY)
93 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
94 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
99 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
102 /* Size of the L2 (and L3, etc) page tables. */
104 #define V_L2_SIZE (1 << V_L2_BITS)
106 /* The bits remaining after N lower levels of page tables. */
107 #define V_L1_BITS_REM \
108 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
110 #if V_L1_BITS_REM < 4
111 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
113 #define V_L1_BITS V_L1_BITS_REM
116 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
118 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
120 uintptr_t qemu_real_host_page_size
;
121 uintptr_t qemu_real_host_page_mask
;
122 uintptr_t qemu_host_page_size
;
123 uintptr_t qemu_host_page_mask
;
125 /* This is a multi-level map on the virtual address space.
126 The bottom level has pointers to PageDesc. */
127 static void *l1_map
[V_L1_SIZE
];
129 /* code generation context */
132 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
133 tb_page_addr_t phys_page2
);
134 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
136 void cpu_gen_init(void)
138 tcg_context_init(&tcg_ctx
);
141 /* return non zero if the very first instruction is invalid so that
142 the virtual CPU can trigger an exception.
144 '*gen_code_size_ptr' contains the size of the generated code (host
147 int cpu_gen_code(CPUArchState
*env
, TranslationBlock
*tb
, int *gen_code_size_ptr
)
149 TCGContext
*s
= &tcg_ctx
;
150 tcg_insn_unit
*gen_code_buf
;
152 #ifdef CONFIG_PROFILER
156 #ifdef CONFIG_PROFILER
157 s
->tb_count1
++; /* includes aborted translations because of
159 ti
= profile_getclock();
163 gen_intermediate_code(env
, tb
);
165 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
167 /* generate machine code */
168 gen_code_buf
= tb
->tc_ptr
;
169 tb
->tb_next_offset
[0] = 0xffff;
170 tb
->tb_next_offset
[1] = 0xffff;
171 s
->tb_next_offset
= tb
->tb_next_offset
;
172 #ifdef USE_DIRECT_JUMP
173 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
176 s
->tb_jmp_offset
= NULL
;
177 s
->tb_next
= tb
->tb_next
;
180 #ifdef CONFIG_PROFILER
182 s
->interm_time
+= profile_getclock() - ti
;
183 s
->code_time
-= profile_getclock();
185 gen_code_size
= tcg_gen_code(s
, gen_code_buf
);
186 *gen_code_size_ptr
= gen_code_size
;
187 #ifdef CONFIG_PROFILER
188 s
->code_time
+= profile_getclock();
189 s
->code_in_len
+= tb
->size
;
190 s
->code_out_len
+= gen_code_size
;
194 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
195 qemu_log("OUT: [size=%d]\n", gen_code_size
);
196 log_disas(tb
->tc_ptr
, gen_code_size
);
204 /* The cpu state corresponding to 'searched_pc' is restored.
206 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
207 uintptr_t searched_pc
)
209 CPUArchState
*env
= cpu
->env_ptr
;
210 TCGContext
*s
= &tcg_ctx
;
213 #ifdef CONFIG_PROFILER
217 #ifdef CONFIG_PROFILER
218 ti
= profile_getclock();
222 gen_intermediate_code_pc(env
, tb
);
224 if (tb
->cflags
& CF_USE_ICOUNT
) {
225 /* Reset the cycle counter to the start of the block. */
226 cpu
->icount_decr
.u16
.low
+= tb
->icount
;
227 /* Clear the IO flag. */
231 /* find opc index corresponding to search_pc */
232 tc_ptr
= (uintptr_t)tb
->tc_ptr
;
233 if (searched_pc
< tc_ptr
)
236 s
->tb_next_offset
= tb
->tb_next_offset
;
237 #ifdef USE_DIRECT_JUMP
238 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
241 s
->tb_jmp_offset
= NULL
;
242 s
->tb_next
= tb
->tb_next
;
244 j
= tcg_gen_code_search_pc(s
, (tcg_insn_unit
*)tc_ptr
,
245 searched_pc
- tc_ptr
);
248 /* now find start of instruction before */
249 while (s
->gen_opc_instr_start
[j
] == 0) {
252 cpu
->icount_decr
.u16
.low
-= s
->gen_opc_icount
[j
];
254 restore_state_to_opc(env
, tb
, j
);
256 #ifdef CONFIG_PROFILER
257 s
->restore_time
+= profile_getclock() - ti
;
263 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
265 TranslationBlock
*tb
;
267 tb
= tb_find_pc(retaddr
);
269 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
270 if (tb
->cflags
& CF_NOCACHE
) {
271 /* one-shot translation, invalidate it immediately */
272 cpu
->current_tb
= NULL
;
273 tb_phys_invalidate(tb
, -1);
282 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
285 VirtualProtect(addr
, size
,
286 PAGE_EXECUTE_READWRITE
, &old_protect
);
289 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
291 unsigned long start
, end
, page_size
;
293 page_size
= getpagesize();
294 start
= (unsigned long)addr
;
295 start
&= ~(page_size
- 1);
297 end
= (unsigned long)addr
+ size
;
298 end
+= page_size
- 1;
299 end
&= ~(page_size
- 1);
301 mprotect((void *)start
, end
- start
,
302 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
306 void page_size_init(void)
308 /* NOTE: we can always suppose that qemu_host_page_size >=
310 qemu_real_host_page_size
= getpagesize();
311 qemu_real_host_page_mask
= ~(qemu_real_host_page_size
- 1);
312 if (qemu_host_page_size
== 0) {
313 qemu_host_page_size
= qemu_real_host_page_size
;
315 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
316 qemu_host_page_size
= TARGET_PAGE_SIZE
;
318 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
321 static void page_init(void)
324 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
326 #ifdef HAVE_KINFO_GETVMMAP
327 struct kinfo_vmentry
*freep
;
330 freep
= kinfo_getvmmap(getpid(), &cnt
);
333 for (i
= 0; i
< cnt
; i
++) {
334 unsigned long startaddr
, endaddr
;
336 startaddr
= freep
[i
].kve_start
;
337 endaddr
= freep
[i
].kve_end
;
338 if (h2g_valid(startaddr
)) {
339 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
341 if (h2g_valid(endaddr
)) {
342 endaddr
= h2g(endaddr
);
343 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
345 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
347 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
358 last_brk
= (unsigned long)sbrk(0);
360 f
= fopen("/compat/linux/proc/self/maps", "r");
365 unsigned long startaddr
, endaddr
;
368 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
370 if (n
== 2 && h2g_valid(startaddr
)) {
371 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
373 if (h2g_valid(endaddr
)) {
374 endaddr
= h2g(endaddr
);
378 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
390 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
396 /* Level 1. Always allocated. */
397 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
400 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
407 p
= g_new0(void *, V_L2_SIZE
);
411 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
419 pd
= g_new0(PageDesc
, V_L2_SIZE
);
423 return pd
+ (index
& (V_L2_SIZE
- 1));
426 static inline PageDesc
*page_find(tb_page_addr_t index
)
428 return page_find_alloc(index
, 0);
431 #if !defined(CONFIG_USER_ONLY)
432 #define mmap_lock() do { } while (0)
433 #define mmap_unlock() do { } while (0)
436 #if defined(CONFIG_USER_ONLY)
437 /* Currently it is not recommended to allocate big chunks of data in
438 user mode. It will change when a dedicated libc will be used. */
439 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
440 region in which the guest needs to run. Revisit this. */
441 #define USE_STATIC_CODE_GEN_BUFFER
444 /* ??? Should configure for this, not list operating systems here. */
445 #if (defined(__linux__) \
446 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
447 || defined(__DragonFly__) || defined(__OpenBSD__) \
448 || defined(__NetBSD__))
452 /* Minimum size of the code gen buffer. This number is randomly chosen,
453 but not so small that we can't have a fair number of TB's live. */
454 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
456 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
457 indicated, this is constrained by the range of direct branches on the
458 host cpu, as used by the TCG implementation of goto_tb. */
459 #if defined(__x86_64__)
460 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
461 #elif defined(__sparc__)
462 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
463 #elif defined(__aarch64__)
464 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
465 #elif defined(__arm__)
466 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
467 #elif defined(__s390x__)
468 /* We have a +- 4GB range on the branches; leave some slop. */
469 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
470 #elif defined(__mips__)
471 /* We have a 256MB branch region, but leave room to make sure the
472 main executable is also within that region. */
473 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
475 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
478 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
480 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
481 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
482 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
484 static inline size_t size_code_gen_buffer(size_t tb_size
)
486 /* Size the buffer. */
488 #ifdef USE_STATIC_CODE_GEN_BUFFER
489 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
491 /* ??? Needs adjustments. */
492 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
493 static buffer, we could size this on RESERVED_VA, on the text
494 segment size of the executable, or continue to use the default. */
495 tb_size
= (unsigned long)(ram_size
/ 4);
498 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
499 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
501 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
502 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
504 tcg_ctx
.code_gen_buffer_size
= tb_size
;
509 /* In order to use J and JAL within the code_gen_buffer, we require
510 that the buffer not cross a 256MB boundary. */
511 static inline bool cross_256mb(void *addr
, size_t size
)
513 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & 0xf0000000;
516 /* We weren't able to allocate a buffer without crossing that boundary,
517 so make do with the larger portion of the buffer that doesn't cross.
518 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
519 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
521 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & 0xf0000000);
522 size_t size2
= buf1
+ size1
- buf2
;
530 tcg_ctx
.code_gen_buffer_size
= size1
;
535 #ifdef USE_STATIC_CODE_GEN_BUFFER
536 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
537 __attribute__((aligned(CODE_GEN_ALIGN
)));
539 static inline void *alloc_code_gen_buffer(void)
541 void *buf
= static_code_gen_buffer
;
543 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
544 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
547 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
550 #elif defined(USE_MMAP)
551 static inline void *alloc_code_gen_buffer(void)
553 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
557 /* Constrain the position of the buffer based on the host cpu.
558 Note that these addresses are chosen in concert with the
559 addresses assigned in the relevant linker script file. */
560 # if defined(__PIE__) || defined(__PIC__)
561 /* Don't bother setting a preferred location if we're building
562 a position-independent executable. We're more likely to get
563 an address near the main executable if we let the kernel
564 choose the address. */
565 # elif defined(__x86_64__) && defined(MAP_32BIT)
566 /* Force the memory down into low memory with the executable.
567 Leave the choice of exact location with the kernel. */
569 /* Cannot expect to map more than 800MB in low memory. */
570 if (tcg_ctx
.code_gen_buffer_size
> 800u * 1024 * 1024) {
571 tcg_ctx
.code_gen_buffer_size
= 800u * 1024 * 1024;
573 # elif defined(__sparc__)
574 start
= 0x40000000ul
;
575 # elif defined(__s390x__)
576 start
= 0x90000000ul
;
577 # elif defined(__mips__)
578 /* ??? We ought to more explicitly manage layout for softmmu too. */
579 # ifdef CONFIG_USER_ONLY
580 start
= 0x68000000ul
;
581 # elif _MIPS_SIM == _ABI64
582 start
= 0x128000000ul
;
584 start
= 0x08000000ul
;
588 buf
= mmap((void *)start
, tcg_ctx
.code_gen_buffer_size
,
589 PROT_WRITE
| PROT_READ
| PROT_EXEC
, flags
, -1, 0);
590 if (buf
== MAP_FAILED
) {
595 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
596 /* Try again, with the original still mapped, to avoid re-acquiring
597 that 256mb crossing. This time don't specify an address. */
598 size_t size2
, size1
= tcg_ctx
.code_gen_buffer_size
;
599 void *buf2
= mmap(NULL
, size1
, PROT_WRITE
| PROT_READ
| PROT_EXEC
,
601 if (buf2
!= MAP_FAILED
) {
602 if (!cross_256mb(buf2
, size1
)) {
603 /* Success! Use the new buffer. */
607 /* Failure. Work with what we had. */
611 /* Split the original buffer. Free the smaller half. */
612 buf2
= split_cross_256mb(buf
, size1
);
613 size2
= tcg_ctx
.code_gen_buffer_size
;
614 munmap(buf
+ (buf
== buf2
? size2
: 0), size1
- size2
);
622 static inline void *alloc_code_gen_buffer(void)
624 void *buf
= g_try_malloc(tcg_ctx
.code_gen_buffer_size
);
631 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
632 void *buf2
= g_malloc(tcg_ctx
.code_gen_buffer_size
);
633 if (buf2
!= NULL
&& !cross_256mb(buf2
, size1
)) {
634 /* Success! Use the new buffer. */
638 /* Failure. Work with what we had. Since this is malloc
639 and not mmap, we can't free the other half. */
641 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
646 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
649 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
651 static inline void code_gen_alloc(size_t tb_size
)
653 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
654 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
655 if (tcg_ctx
.code_gen_buffer
== NULL
) {
656 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
660 qemu_madvise(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
,
663 /* Steal room for the prologue at the end of the buffer. This ensures
664 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
665 from TB's to the prologue are going to be in range. It also means
666 that we don't need to mark (additional) portions of the data segment
668 tcg_ctx
.code_gen_prologue
= tcg_ctx
.code_gen_buffer
+
669 tcg_ctx
.code_gen_buffer_size
- 1024;
670 tcg_ctx
.code_gen_buffer_size
-= 1024;
672 tcg_ctx
.code_gen_buffer_max_size
= tcg_ctx
.code_gen_buffer_size
-
673 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
674 tcg_ctx
.code_gen_max_blocks
= tcg_ctx
.code_gen_buffer_size
/
675 CODE_GEN_AVG_BLOCK_SIZE
;
677 g_malloc(tcg_ctx
.code_gen_max_blocks
* sizeof(TranslationBlock
));
680 /* Must be called before using the QEMU cpus. 'tb_size' is the size
681 (in bytes) allocated to the translation buffer. Zero means default
683 void tcg_exec_init(uintptr_t tb_size
)
686 code_gen_alloc(tb_size
);
687 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
688 tcg_register_jit(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
);
690 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
691 /* There's no guest base to take into account, so go ahead and
692 initialize the prologue now. */
693 tcg_prologue_init(&tcg_ctx
);
697 bool tcg_enabled(void)
699 return tcg_ctx
.code_gen_buffer
!= NULL
;
702 /* Allocate a new translation block. Flush the translation buffer if
703 too many translation blocks or too much generated code. */
704 static TranslationBlock
*tb_alloc(target_ulong pc
)
706 TranslationBlock
*tb
;
708 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
||
709 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) >=
710 tcg_ctx
.code_gen_buffer_max_size
) {
713 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
719 void tb_free(TranslationBlock
*tb
)
721 /* In practice this is mostly used for single use temporary TB
722 Ignore the hard cases and just back up if this TB happens to
723 be the last one generated. */
724 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
725 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
726 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
727 tcg_ctx
.tb_ctx
.nb_tbs
--;
731 static inline void invalidate_page_bitmap(PageDesc
*p
)
733 if (p
->code_bitmap
) {
734 g_free(p
->code_bitmap
);
735 p
->code_bitmap
= NULL
;
737 p
->code_write_count
= 0;
740 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
741 static void page_flush_tb_1(int level
, void **lp
)
751 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
752 pd
[i
].first_tb
= NULL
;
753 invalidate_page_bitmap(pd
+ i
);
758 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
759 page_flush_tb_1(level
- 1, pp
+ i
);
764 static void page_flush_tb(void)
768 for (i
= 0; i
< V_L1_SIZE
; i
++) {
769 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
773 /* flush all the translation blocks */
774 /* XXX: tb_flush is currently not thread safe */
775 void tb_flush(CPUState
*cpu
)
777 #if defined(DEBUG_FLUSH)
778 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
779 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
780 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
781 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
782 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
784 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
785 > tcg_ctx
.code_gen_buffer_size
) {
786 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
788 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
791 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
794 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
797 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
798 /* XXX: flush processor icache at this point if cache flush is
800 tcg_ctx
.tb_ctx
.tb_flush_count
++;
803 #ifdef DEBUG_TB_CHECK
805 static void tb_invalidate_check(target_ulong address
)
807 TranslationBlock
*tb
;
810 address
&= TARGET_PAGE_MASK
;
811 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
812 for (tb
= tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
813 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
814 address
>= tb
->pc
+ tb
->size
)) {
815 printf("ERROR invalidate: address=" TARGET_FMT_lx
816 " PC=%08lx size=%04x\n",
817 address
, (long)tb
->pc
, tb
->size
);
823 /* verify that all the pages have correct rights for code */
824 static void tb_page_check(void)
826 TranslationBlock
*tb
;
827 int i
, flags1
, flags2
;
829 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
830 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
831 tb
= tb
->phys_hash_next
) {
832 flags1
= page_get_flags(tb
->pc
);
833 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
834 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
835 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
836 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
844 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
846 TranslationBlock
*tb1
;
851 *ptb
= tb1
->phys_hash_next
;
854 ptb
= &tb1
->phys_hash_next
;
858 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
860 TranslationBlock
*tb1
;
865 n1
= (uintptr_t)tb1
& 3;
866 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
868 *ptb
= tb1
->page_next
[n1
];
871 ptb
= &tb1
->page_next
[n1
];
875 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
877 TranslationBlock
*tb1
, **ptb
;
880 ptb
= &tb
->jmp_next
[n
];
883 /* find tb(n) in circular list */
886 n1
= (uintptr_t)tb1
& 3;
887 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
888 if (n1
== n
&& tb1
== tb
) {
892 ptb
= &tb1
->jmp_first
;
894 ptb
= &tb1
->jmp_next
[n1
];
897 /* now we can suppress tb(n) from the list */
898 *ptb
= tb
->jmp_next
[n
];
900 tb
->jmp_next
[n
] = NULL
;
904 /* reset the jump entry 'n' of a TB so that it is not chained to
906 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
908 tb_set_jmp_target(tb
, n
, (uintptr_t)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
911 /* invalidate one TB */
912 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
917 tb_page_addr_t phys_pc
;
918 TranslationBlock
*tb1
, *tb2
;
920 /* remove the TB from the hash list */
921 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
922 h
= tb_phys_hash_func(phys_pc
);
923 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
925 /* remove the TB from the page list */
926 if (tb
->page_addr
[0] != page_addr
) {
927 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
928 tb_page_remove(&p
->first_tb
, tb
);
929 invalidate_page_bitmap(p
);
931 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
932 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
933 tb_page_remove(&p
->first_tb
, tb
);
934 invalidate_page_bitmap(p
);
937 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
939 /* remove the TB from the hash list */
940 h
= tb_jmp_cache_hash_func(tb
->pc
);
942 if (cpu
->tb_jmp_cache
[h
] == tb
) {
943 cpu
->tb_jmp_cache
[h
] = NULL
;
947 /* suppress this TB from the two jump lists */
948 tb_jmp_remove(tb
, 0);
949 tb_jmp_remove(tb
, 1);
951 /* suppress any remaining jumps to this TB */
954 n1
= (uintptr_t)tb1
& 3;
958 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
959 tb2
= tb1
->jmp_next
[n1
];
960 tb_reset_jump(tb1
, n1
);
961 tb1
->jmp_next
[n1
] = NULL
;
964 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2); /* fail safe */
966 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
969 static void build_page_bitmap(PageDesc
*p
)
971 int n
, tb_start
, tb_end
;
972 TranslationBlock
*tb
;
974 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
978 n
= (uintptr_t)tb
& 3;
979 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
980 /* NOTE: this is subtle as a TB may span two physical pages */
982 /* NOTE: tb_end may be after the end of the page, but
983 it is not a problem */
984 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
985 tb_end
= tb_start
+ tb
->size
;
986 if (tb_end
> TARGET_PAGE_SIZE
) {
987 tb_end
= TARGET_PAGE_SIZE
;
991 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
993 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
994 tb
= tb
->page_next
[n
];
998 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
999 target_ulong pc
, target_ulong cs_base
,
1000 int flags
, int cflags
)
1002 CPUArchState
*env
= cpu
->env_ptr
;
1003 TranslationBlock
*tb
;
1004 tb_page_addr_t phys_pc
, phys_page2
;
1005 target_ulong virt_page2
;
1008 phys_pc
= get_page_addr_code(env
, pc
);
1010 cflags
|= CF_USE_ICOUNT
;
1014 /* flush must be done */
1016 /* cannot fail at this point */
1018 /* Don't forget to invalidate previous TB info. */
1019 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
1021 tb
->tc_ptr
= tcg_ctx
.code_gen_ptr
;
1022 tb
->cs_base
= cs_base
;
1024 tb
->cflags
= cflags
;
1025 cpu_gen_code(env
, tb
, &code_gen_size
);
1026 tcg_ctx
.code_gen_ptr
= (void *)(((uintptr_t)tcg_ctx
.code_gen_ptr
+
1027 code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1029 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1030 /* if we are doing vsyscall don't link the page as it lies in high memory
1031 and tb_alloc_page will abort due to page_l1_map returning NULL */
1032 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1033 && phys_pc
< TARGET_VSYSCALL_END
))
1037 /* check next page if needed */
1038 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1040 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1041 phys_page2
= get_page_addr_code(env
, virt_page2
);
1043 tb_link_page(tb
, phys_pc
, phys_page2
);
1048 * Invalidate all TBs which intersect with the target physical address range
1049 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1050 * 'is_cpu_write_access' should be true if called from a real cpu write
1051 * access: the virtual CPU will exit the current TB if code is modified inside
1054 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1056 while (start
< end
) {
1057 tb_invalidate_phys_page_range(start
, end
, 0);
1058 start
&= TARGET_PAGE_MASK
;
1059 start
+= TARGET_PAGE_SIZE
;
1064 * Invalidate all TBs which intersect with the target physical address range
1065 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1066 * 'is_cpu_write_access' should be true if called from a real cpu write
1067 * access: the virtual CPU will exit the current TB if code is modified inside
1070 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1071 int is_cpu_write_access
)
1073 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1074 CPUState
*cpu
= current_cpu
;
1075 #if defined(TARGET_HAS_PRECISE_SMC)
1076 CPUArchState
*env
= NULL
;
1078 tb_page_addr_t tb_start
, tb_end
;
1081 #ifdef TARGET_HAS_PRECISE_SMC
1082 int current_tb_not_found
= is_cpu_write_access
;
1083 TranslationBlock
*current_tb
= NULL
;
1084 int current_tb_modified
= 0;
1085 target_ulong current_pc
= 0;
1086 target_ulong current_cs_base
= 0;
1087 int current_flags
= 0;
1088 #endif /* TARGET_HAS_PRECISE_SMC */
1090 p
= page_find(start
>> TARGET_PAGE_BITS
);
1094 #if defined(TARGET_HAS_PRECISE_SMC)
1100 /* we remove all the TBs in the range [start, end[ */
1101 /* XXX: see if in some cases it could be faster to invalidate all
1104 while (tb
!= NULL
) {
1105 n
= (uintptr_t)tb
& 3;
1106 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1107 tb_next
= tb
->page_next
[n
];
1108 /* NOTE: this is subtle as a TB may span two physical pages */
1110 /* NOTE: tb_end may be after the end of the page, but
1111 it is not a problem */
1112 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1113 tb_end
= tb_start
+ tb
->size
;
1115 tb_start
= tb
->page_addr
[1];
1116 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1118 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1119 #ifdef TARGET_HAS_PRECISE_SMC
1120 if (current_tb_not_found
) {
1121 current_tb_not_found
= 0;
1123 if (cpu
->mem_io_pc
) {
1124 /* now we have a real cpu fault */
1125 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1128 if (current_tb
== tb
&&
1129 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1130 /* If we are modifying the current TB, we must stop
1131 its execution. We could be more precise by checking
1132 that the modification is after the current PC, but it
1133 would require a specialized function to partially
1134 restore the CPU state */
1136 current_tb_modified
= 1;
1137 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1138 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1141 #endif /* TARGET_HAS_PRECISE_SMC */
1142 /* we need to do that to handle the case where a signal
1143 occurs while doing tb_phys_invalidate() */
1146 saved_tb
= cpu
->current_tb
;
1147 cpu
->current_tb
= NULL
;
1149 tb_phys_invalidate(tb
, -1);
1151 cpu
->current_tb
= saved_tb
;
1152 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1153 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1159 #if !defined(CONFIG_USER_ONLY)
1160 /* if no code remaining, no need to continue to use slow writes */
1162 invalidate_page_bitmap(p
);
1163 tlb_unprotect_code(start
);
1166 #ifdef TARGET_HAS_PRECISE_SMC
1167 if (current_tb_modified
) {
1168 /* we generate a block containing just the instruction
1169 modifying the memory. It will ensure that it cannot modify
1171 cpu
->current_tb
= NULL
;
1172 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1173 cpu_resume_from_signal(cpu
, NULL
);
1178 /* len must be <= 8 and start must be a multiple of len */
1179 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1185 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1186 cpu_single_env
->mem_io_vaddr
, len
,
1187 cpu_single_env
->eip
,
1188 cpu_single_env
->eip
+
1189 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1192 p
= page_find(start
>> TARGET_PAGE_BITS
);
1196 if (!p
->code_bitmap
&&
1197 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1198 /* build code bitmap */
1199 build_page_bitmap(p
);
1201 if (p
->code_bitmap
) {
1205 nr
= start
& ~TARGET_PAGE_MASK
;
1206 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1207 if (b
& ((1 << len
) - 1)) {
1212 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1216 #if !defined(CONFIG_SOFTMMU)
1217 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1218 uintptr_t pc
, void *puc
,
1221 TranslationBlock
*tb
;
1224 #ifdef TARGET_HAS_PRECISE_SMC
1225 TranslationBlock
*current_tb
= NULL
;
1226 CPUState
*cpu
= current_cpu
;
1227 CPUArchState
*env
= NULL
;
1228 int current_tb_modified
= 0;
1229 target_ulong current_pc
= 0;
1230 target_ulong current_cs_base
= 0;
1231 int current_flags
= 0;
1234 addr
&= TARGET_PAGE_MASK
;
1235 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1240 #ifdef TARGET_HAS_PRECISE_SMC
1241 if (tb
&& pc
!= 0) {
1242 current_tb
= tb_find_pc(pc
);
1248 while (tb
!= NULL
) {
1249 n
= (uintptr_t)tb
& 3;
1250 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1251 #ifdef TARGET_HAS_PRECISE_SMC
1252 if (current_tb
== tb
&&
1253 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1254 /* If we are modifying the current TB, we must stop
1255 its execution. We could be more precise by checking
1256 that the modification is after the current PC, but it
1257 would require a specialized function to partially
1258 restore the CPU state */
1260 current_tb_modified
= 1;
1261 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1262 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1265 #endif /* TARGET_HAS_PRECISE_SMC */
1266 tb_phys_invalidate(tb
, addr
);
1267 tb
= tb
->page_next
[n
];
1270 #ifdef TARGET_HAS_PRECISE_SMC
1271 if (current_tb_modified
) {
1272 /* we generate a block containing just the instruction
1273 modifying the memory. It will ensure that it cannot modify
1275 cpu
->current_tb
= NULL
;
1276 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1280 cpu_resume_from_signal(cpu
, puc
);
1286 /* add the tb in the target page and protect it if necessary */
1287 static inline void tb_alloc_page(TranslationBlock
*tb
,
1288 unsigned int n
, tb_page_addr_t page_addr
)
1291 #ifndef CONFIG_USER_ONLY
1292 bool page_already_protected
;
1295 tb
->page_addr
[n
] = page_addr
;
1296 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1297 tb
->page_next
[n
] = p
->first_tb
;
1298 #ifndef CONFIG_USER_ONLY
1299 page_already_protected
= p
->first_tb
!= NULL
;
1301 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1302 invalidate_page_bitmap(p
);
1304 #if defined(CONFIG_USER_ONLY)
1305 if (p
->flags
& PAGE_WRITE
) {
1310 /* force the host page as non writable (writes will have a
1311 page fault + mprotect overhead) */
1312 page_addr
&= qemu_host_page_mask
;
1314 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1315 addr
+= TARGET_PAGE_SIZE
) {
1317 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1322 p2
->flags
&= ~PAGE_WRITE
;
1324 mprotect(g2h(page_addr
), qemu_host_page_size
,
1325 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1326 #ifdef DEBUG_TB_INVALIDATE
1327 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1332 /* if some code is already present, then the pages are already
1333 protected. So we handle the case where only the first TB is
1334 allocated in a physical page */
1335 if (!page_already_protected
) {
1336 tlb_protect_code(page_addr
);
1341 /* add a new TB and link it to the physical page tables. phys_page2 is
1342 (-1) to indicate that only one page contains the TB. */
1343 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1344 tb_page_addr_t phys_page2
)
1347 TranslationBlock
**ptb
;
1349 /* Grab the mmap lock to stop another thread invalidating this TB
1350 before we are done. */
1352 /* add in the physical hash table */
1353 h
= tb_phys_hash_func(phys_pc
);
1354 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1355 tb
->phys_hash_next
= *ptb
;
1358 /* add in the page list */
1359 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1360 if (phys_page2
!= -1) {
1361 tb_alloc_page(tb
, 1, phys_page2
);
1363 tb
->page_addr
[1] = -1;
1366 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2);
1367 tb
->jmp_next
[0] = NULL
;
1368 tb
->jmp_next
[1] = NULL
;
1370 /* init original jump addresses */
1371 if (tb
->tb_next_offset
[0] != 0xffff) {
1372 tb_reset_jump(tb
, 0);
1374 if (tb
->tb_next_offset
[1] != 0xffff) {
1375 tb_reset_jump(tb
, 1);
1378 #ifdef DEBUG_TB_CHECK
1384 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1385 tb[1].tc_ptr. Return NULL if not found */
1386 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1388 int m_min
, m_max
, m
;
1390 TranslationBlock
*tb
;
1392 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1395 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1396 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1399 /* binary search (cf Knuth) */
1401 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1402 while (m_min
<= m_max
) {
1403 m
= (m_min
+ m_max
) >> 1;
1404 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1405 v
= (uintptr_t)tb
->tc_ptr
;
1408 } else if (tc_ptr
< v
) {
1414 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1417 #if !defined(CONFIG_USER_ONLY)
1418 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1420 ram_addr_t ram_addr
;
1425 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1426 if (!(memory_region_is_ram(mr
)
1427 || memory_region_is_romd(mr
))) {
1431 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1433 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1436 #endif /* !defined(CONFIG_USER_ONLY) */
1438 void tb_check_watchpoint(CPUState
*cpu
)
1440 TranslationBlock
*tb
;
1442 tb
= tb_find_pc(cpu
->mem_io_pc
);
1444 /* We can use retranslation to find the PC. */
1445 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1446 tb_phys_invalidate(tb
, -1);
1448 /* The exception probably happened in a helper. The CPU state should
1449 have been saved before calling it. Fetch the PC from there. */
1450 CPUArchState
*env
= cpu
->env_ptr
;
1451 target_ulong pc
, cs_base
;
1452 tb_page_addr_t addr
;
1455 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1456 addr
= get_page_addr_code(env
, pc
);
1457 tb_invalidate_phys_range(addr
, addr
+ 1);
1461 #ifndef CONFIG_USER_ONLY
1462 /* mask must never be zero, except for A20 change call */
1463 static void tcg_handle_interrupt(CPUState
*cpu
, int mask
)
1467 old_mask
= cpu
->interrupt_request
;
1468 cpu
->interrupt_request
|= mask
;
1471 * If called from iothread context, wake the target cpu in
1474 if (!qemu_cpu_is_self(cpu
)) {
1480 cpu
->icount_decr
.u16
.high
= 0xffff;
1481 if (!cpu_can_do_io(cpu
)
1482 && (mask
& ~old_mask
) != 0) {
1483 cpu_abort(cpu
, "Raised interrupt while not in I/O function");
1486 cpu
->tcg_exit_req
= 1;
1490 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1492 /* in deterministic execution mode, instructions doing device I/Os
1493 must be at the end of the TB */
1494 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1496 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1497 CPUArchState
*env
= cpu
->env_ptr
;
1499 TranslationBlock
*tb
;
1501 target_ulong pc
, cs_base
;
1504 tb
= tb_find_pc(retaddr
);
1506 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1509 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1510 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1511 /* Calculate how many instructions had been executed before the fault
1513 n
= n
- cpu
->icount_decr
.u16
.low
;
1514 /* Generate a new TB ending on the I/O insn. */
1516 /* On MIPS and SH, delay slot instructions can only be restarted if
1517 they were already the first instruction in the TB. If this is not
1518 the first instruction in a TB then re-execute the preceding
1520 #if defined(TARGET_MIPS)
1521 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1522 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1523 cpu
->icount_decr
.u16
.low
++;
1524 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1526 #elif defined(TARGET_SH4)
1527 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1530 cpu
->icount_decr
.u16
.low
++;
1531 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1534 /* This should never happen. */
1535 if (n
> CF_COUNT_MASK
) {
1536 cpu_abort(cpu
, "TB too big during recompile");
1539 cflags
= n
| CF_LAST_IO
;
1541 cs_base
= tb
->cs_base
;
1543 tb_phys_invalidate(tb
, -1);
1544 /* FIXME: In theory this could raise an exception. In practice
1545 we have already translated the block once so it's probably ok. */
1546 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1547 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1548 the first in the TB) then we end up generating a whole new TB and
1549 repeating the fault, which is horribly inefficient.
1550 Better would be to execute just this insn uncached, or generate a
1552 cpu_resume_from_signal(cpu
, NULL
);
1555 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1559 /* Discard jump cache entries for any tb which might potentially
1560 overlap the flushed page. */
1561 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1562 memset(&cpu
->tb_jmp_cache
[i
], 0,
1563 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1565 i
= tb_jmp_cache_hash_page(addr
);
1566 memset(&cpu
->tb_jmp_cache
[i
], 0,
1567 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1570 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1572 int i
, target_code_size
, max_target_code_size
;
1573 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1574 TranslationBlock
*tb
;
1576 target_code_size
= 0;
1577 max_target_code_size
= 0;
1579 direct_jmp_count
= 0;
1580 direct_jmp2_count
= 0;
1581 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1582 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1583 target_code_size
+= tb
->size
;
1584 if (tb
->size
> max_target_code_size
) {
1585 max_target_code_size
= tb
->size
;
1587 if (tb
->page_addr
[1] != -1) {
1590 if (tb
->tb_next_offset
[0] != 0xffff) {
1592 if (tb
->tb_next_offset
[1] != 0xffff) {
1593 direct_jmp2_count
++;
1597 /* XXX: avoid using doubles ? */
1598 cpu_fprintf(f
, "Translation buffer state:\n");
1599 cpu_fprintf(f
, "gen code size %td/%zd\n",
1600 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1601 tcg_ctx
.code_gen_buffer_max_size
);
1602 cpu_fprintf(f
, "TB count %d/%d\n",
1603 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1604 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1605 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1606 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1607 max_target_code_size
);
1608 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1609 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1610 tcg_ctx
.code_gen_buffer
) /
1611 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1612 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1613 tcg_ctx
.code_gen_buffer
) /
1614 target_code_size
: 0);
1615 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1616 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1617 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1618 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1620 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1621 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1623 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1624 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1625 cpu_fprintf(f
, "\nStatistics:\n");
1626 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1627 cpu_fprintf(f
, "TB invalidate count %d\n",
1628 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1629 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1630 tcg_dump_info(f
, cpu_fprintf
);
1633 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1635 tcg_dump_op_count(f
, cpu_fprintf
);
1638 #else /* CONFIG_USER_ONLY */
1640 void cpu_interrupt(CPUState
*cpu
, int mask
)
1642 cpu
->interrupt_request
|= mask
;
1643 cpu
->tcg_exit_req
= 1;
1647 * Walks guest process memory "regions" one by one
1648 * and calls callback function 'fn' for each region.
1650 struct walk_memory_regions_data
{
1651 walk_memory_regions_fn fn
;
1657 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1658 target_ulong end
, int new_prot
)
1660 if (data
->start
!= -1u) {
1661 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1667 data
->start
= (new_prot
? end
: -1u);
1668 data
->prot
= new_prot
;
1673 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1674 target_ulong base
, int level
, void **lp
)
1680 return walk_memory_regions_end(data
, base
, 0);
1686 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1687 int prot
= pd
[i
].flags
;
1689 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1690 if (prot
!= data
->prot
) {
1691 rc
= walk_memory_regions_end(data
, pa
, prot
);
1700 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1701 pa
= base
| ((target_ulong
)i
<<
1702 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1703 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1713 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1715 struct walk_memory_regions_data data
;
1723 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1724 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1725 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1731 return walk_memory_regions_end(&data
, 0, 0);
1734 static int dump_region(void *priv
, target_ulong start
,
1735 target_ulong end
, abi_ulong prot
)
1737 FILE *f
= (FILE *)priv
;
1739 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1740 " "TARGET_FMT_lx
" %c%c%c\n",
1741 start
, end
, end
- start
,
1742 ((prot
& PAGE_READ
) ? 'r' : '-'),
1743 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1744 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1749 /* dump memory mappings */
1750 void page_dump(FILE *f
)
1752 const int length
= sizeof(target_ulong
) * 2;
1753 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1754 length
, "start", length
, "end", length
, "size", "prot");
1755 walk_memory_regions(f
, dump_region
);
1758 int page_get_flags(target_ulong address
)
1762 p
= page_find(address
>> TARGET_PAGE_BITS
);
1769 /* Modify the flags of a page and invalidate the code if necessary.
1770 The flag PAGE_WRITE_ORG is positioned automatically depending
1771 on PAGE_WRITE. The mmap_lock should already be held. */
1772 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1774 target_ulong addr
, len
;
1776 /* This function should never be called with addresses outside the
1777 guest address space. If this assert fires, it probably indicates
1778 a missing call to h2g_valid. */
1779 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1780 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1782 assert(start
< end
);
1784 start
= start
& TARGET_PAGE_MASK
;
1785 end
= TARGET_PAGE_ALIGN(end
);
1787 if (flags
& PAGE_WRITE
) {
1788 flags
|= PAGE_WRITE_ORG
;
1791 for (addr
= start
, len
= end
- start
;
1793 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1794 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1796 /* If the write protection bit is set, then we invalidate
1798 if (!(p
->flags
& PAGE_WRITE
) &&
1799 (flags
& PAGE_WRITE
) &&
1801 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1807 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1813 /* This function should never be called with addresses outside the
1814 guest address space. If this assert fires, it probably indicates
1815 a missing call to h2g_valid. */
1816 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1817 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1823 if (start
+ len
- 1 < start
) {
1824 /* We've wrapped around. */
1828 /* must do before we loose bits in the next step */
1829 end
= TARGET_PAGE_ALIGN(start
+ len
);
1830 start
= start
& TARGET_PAGE_MASK
;
1832 for (addr
= start
, len
= end
- start
;
1834 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1835 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1839 if (!(p
->flags
& PAGE_VALID
)) {
1843 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1846 if (flags
& PAGE_WRITE
) {
1847 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1850 /* unprotect the page if it was put read-only because it
1851 contains translated code */
1852 if (!(p
->flags
& PAGE_WRITE
)) {
1853 if (!page_unprotect(addr
, 0, NULL
)) {
1862 /* called from signal handler: invalidate the code and unprotect the
1863 page. Return TRUE if the fault was successfully handled. */
1864 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1868 target_ulong host_start
, host_end
, addr
;
1870 /* Technically this isn't safe inside a signal handler. However we
1871 know this only ever happens in a synchronous SEGV handler, so in
1872 practice it seems to be ok. */
1875 p
= page_find(address
>> TARGET_PAGE_BITS
);
1881 /* if the page was really writable, then we change its
1882 protection back to writable */
1883 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
1884 host_start
= address
& qemu_host_page_mask
;
1885 host_end
= host_start
+ qemu_host_page_size
;
1888 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
1889 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1890 p
->flags
|= PAGE_WRITE
;
1893 /* and since the content will be modified, we must invalidate
1894 the corresponding translated code. */
1895 tb_invalidate_phys_page(addr
, pc
, puc
, true);
1896 #ifdef DEBUG_TB_CHECK
1897 tb_invalidate_check(addr
);
1900 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
1909 #endif /* CONFIG_USER_ONLY */