4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
30 #include "qemu-common.h"
31 #define NO_CPU_IO_DEFS
33 #include "disas/disas.h"
35 #if defined(CONFIG_USER_ONLY)
37 #if defined(TARGET_X86_64)
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
47 #include <machine/profile.h>
56 #include "exec/address-spaces.h"
59 #include "exec/cputlb.h"
60 #include "translate-all.h"
61 #include "qemu/timer.h"
63 //#define DEBUG_TB_INVALIDATE
65 /* make various TB consistency checks */
66 //#define DEBUG_TB_CHECK
68 #if !defined(CONFIG_USER_ONLY)
69 /* TB consistency checks only implemented for usermode emulation. */
73 #define SMC_BITMAP_USE_THRESHOLD 10
75 typedef struct PageDesc
{
76 /* list of TBs intersecting this ram page */
77 TranslationBlock
*first_tb
;
78 /* in order to optimize self modifying code, we count the number
79 of lookups we do to a given page to use a bitmap */
80 unsigned int code_write_count
;
82 #if defined(CONFIG_USER_ONLY)
87 /* In system mode we want L1_MAP to be based on ram offsets,
88 while in user mode we want it to be based on virtual addresses. */
89 #if !defined(CONFIG_USER_ONLY)
90 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
91 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
99 /* Size of the L2 (and L3, etc) page tables. */
101 #define V_L2_SIZE (1 << V_L2_BITS)
103 /* The bits remaining after N lower levels of page tables. */
104 #define V_L1_BITS_REM \
105 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
107 #if V_L1_BITS_REM < 4
108 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
110 #define V_L1_BITS V_L1_BITS_REM
113 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
115 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
117 uintptr_t qemu_real_host_page_size
;
118 uintptr_t qemu_host_page_size
;
119 uintptr_t qemu_host_page_mask
;
121 /* This is a multi-level map on the virtual address space.
122 The bottom level has pointers to PageDesc. */
123 static void *l1_map
[V_L1_SIZE
];
125 /* code generation context */
128 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
129 tb_page_addr_t phys_page2
);
130 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
132 void cpu_gen_init(void)
134 tcg_context_init(&tcg_ctx
);
137 /* return non zero if the very first instruction is invalid so that
138 the virtual CPU can trigger an exception.
140 '*gen_code_size_ptr' contains the size of the generated code (host
143 int cpu_gen_code(CPUArchState
*env
, TranslationBlock
*tb
, int *gen_code_size_ptr
)
145 TCGContext
*s
= &tcg_ctx
;
146 tcg_insn_unit
*gen_code_buf
;
148 #ifdef CONFIG_PROFILER
152 #ifdef CONFIG_PROFILER
153 s
->tb_count1
++; /* includes aborted translations because of
155 ti
= profile_getclock();
159 gen_intermediate_code(env
, tb
);
161 /* generate machine code */
162 gen_code_buf
= tb
->tc_ptr
;
163 tb
->tb_next_offset
[0] = 0xffff;
164 tb
->tb_next_offset
[1] = 0xffff;
165 s
->tb_next_offset
= tb
->tb_next_offset
;
166 #ifdef USE_DIRECT_JUMP
167 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
170 s
->tb_jmp_offset
= NULL
;
171 s
->tb_next
= tb
->tb_next
;
174 #ifdef CONFIG_PROFILER
176 s
->interm_time
+= profile_getclock() - ti
;
177 s
->code_time
-= profile_getclock();
179 gen_code_size
= tcg_gen_code(s
, gen_code_buf
);
180 *gen_code_size_ptr
= gen_code_size
;
181 #ifdef CONFIG_PROFILER
182 s
->code_time
+= profile_getclock();
183 s
->code_in_len
+= tb
->size
;
184 s
->code_out_len
+= gen_code_size
;
188 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
189 qemu_log("OUT: [size=%d]\n", gen_code_size
);
190 log_disas(tb
->tc_ptr
, gen_code_size
);
198 /* The cpu state corresponding to 'searched_pc' is restored.
200 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
201 uintptr_t searched_pc
)
203 CPUArchState
*env
= cpu
->env_ptr
;
204 TCGContext
*s
= &tcg_ctx
;
207 #ifdef CONFIG_PROFILER
211 #ifdef CONFIG_PROFILER
212 ti
= profile_getclock();
216 gen_intermediate_code_pc(env
, tb
);
219 /* Reset the cycle counter to the start of the block. */
220 cpu
->icount_decr
.u16
.low
+= tb
->icount
;
221 /* Clear the IO flag. */
225 /* find opc index corresponding to search_pc */
226 tc_ptr
= (uintptr_t)tb
->tc_ptr
;
227 if (searched_pc
< tc_ptr
)
230 s
->tb_next_offset
= tb
->tb_next_offset
;
231 #ifdef USE_DIRECT_JUMP
232 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
235 s
->tb_jmp_offset
= NULL
;
236 s
->tb_next
= tb
->tb_next
;
238 j
= tcg_gen_code_search_pc(s
, (tcg_insn_unit
*)tc_ptr
,
239 searched_pc
- tc_ptr
);
242 /* now find start of instruction before */
243 while (s
->gen_opc_instr_start
[j
] == 0) {
246 cpu
->icount_decr
.u16
.low
-= s
->gen_opc_icount
[j
];
248 restore_state_to_opc(env
, tb
, j
);
250 #ifdef CONFIG_PROFILER
251 s
->restore_time
+= profile_getclock() - ti
;
257 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
259 TranslationBlock
*tb
;
261 tb
= tb_find_pc(retaddr
);
263 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
270 static inline void map_exec(void *addr
, long size
)
273 VirtualProtect(addr
, size
,
274 PAGE_EXECUTE_READWRITE
, &old_protect
);
277 static inline void map_exec(void *addr
, long size
)
279 unsigned long start
, end
, page_size
;
281 page_size
= getpagesize();
282 start
= (unsigned long)addr
;
283 start
&= ~(page_size
- 1);
285 end
= (unsigned long)addr
+ size
;
286 end
+= page_size
- 1;
287 end
&= ~(page_size
- 1);
289 mprotect((void *)start
, end
- start
,
290 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
294 void page_size_init(void)
296 /* NOTE: we can always suppose that qemu_host_page_size >=
298 qemu_real_host_page_size
= getpagesize();
299 if (qemu_host_page_size
== 0) {
300 qemu_host_page_size
= qemu_real_host_page_size
;
302 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
303 qemu_host_page_size
= TARGET_PAGE_SIZE
;
305 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
308 static void page_init(void)
311 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
313 #ifdef HAVE_KINFO_GETVMMAP
314 struct kinfo_vmentry
*freep
;
317 freep
= kinfo_getvmmap(getpid(), &cnt
);
320 for (i
= 0; i
< cnt
; i
++) {
321 unsigned long startaddr
, endaddr
;
323 startaddr
= freep
[i
].kve_start
;
324 endaddr
= freep
[i
].kve_end
;
325 if (h2g_valid(startaddr
)) {
326 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
328 if (h2g_valid(endaddr
)) {
329 endaddr
= h2g(endaddr
);
330 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
332 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
334 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
345 last_brk
= (unsigned long)sbrk(0);
347 f
= fopen("/compat/linux/proc/self/maps", "r");
352 unsigned long startaddr
, endaddr
;
355 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
357 if (n
== 2 && h2g_valid(startaddr
)) {
358 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
360 if (h2g_valid(endaddr
)) {
361 endaddr
= h2g(endaddr
);
365 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
377 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
383 #if defined(CONFIG_USER_ONLY)
384 /* We can't use g_malloc because it may recurse into a locked mutex. */
385 # define ALLOC(P, SIZE) \
387 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
388 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
391 # define ALLOC(P, SIZE) \
392 do { P = g_malloc0(SIZE); } while (0)
395 /* Level 1. Always allocated. */
396 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
399 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
406 ALLOC(p
, sizeof(void *) * V_L2_SIZE
);
410 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
418 ALLOC(pd
, sizeof(PageDesc
) * V_L2_SIZE
);
424 return pd
+ (index
& (V_L2_SIZE
- 1));
427 static inline PageDesc
*page_find(tb_page_addr_t index
)
429 return page_find_alloc(index
, 0);
432 #if !defined(CONFIG_USER_ONLY)
433 #define mmap_lock() do { } while (0)
434 #define mmap_unlock() do { } while (0)
437 #if defined(CONFIG_USER_ONLY)
438 /* Currently it is not recommended to allocate big chunks of data in
439 user mode. It will change when a dedicated libc will be used. */
440 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
441 region in which the guest needs to run. Revisit this. */
442 #define USE_STATIC_CODE_GEN_BUFFER
445 /* ??? Should configure for this, not list operating systems here. */
446 #if (defined(__linux__) \
447 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
448 || defined(__DragonFly__) || defined(__OpenBSD__) \
449 || defined(__NetBSD__))
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__aarch64__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
466 #elif defined(__arm__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
468 #elif defined(__s390x__)
469 /* We have a +- 4GB range on the branches; leave some slop. */
470 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
471 #elif defined(__mips__)
472 /* We have a 256MB branch region, but leave room to make sure the
473 main executable is also within that region. */
474 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
476 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
479 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
482 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
483 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
485 static inline size_t size_code_gen_buffer(size_t tb_size
)
487 /* Size the buffer. */
489 #ifdef USE_STATIC_CODE_GEN_BUFFER
490 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
492 /* ??? Needs adjustments. */
493 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
494 static buffer, we could size this on RESERVED_VA, on the text
495 segment size of the executable, or continue to use the default. */
496 tb_size
= (unsigned long)(ram_size
/ 4);
499 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
500 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
502 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
503 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
505 tcg_ctx
.code_gen_buffer_size
= tb_size
;
510 /* In order to use J and JAL within the code_gen_buffer, we require
511 that the buffer not cross a 256MB boundary. */
512 static inline bool cross_256mb(void *addr
, size_t size
)
514 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & 0xf0000000;
517 /* We weren't able to allocate a buffer without crossing that boundary,
518 so make do with the larger portion of the buffer that doesn't cross.
519 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
520 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
522 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & 0xf0000000);
523 size_t size2
= buf1
+ size1
- buf2
;
531 tcg_ctx
.code_gen_buffer_size
= size1
;
536 #ifdef USE_STATIC_CODE_GEN_BUFFER
537 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
538 __attribute__((aligned(CODE_GEN_ALIGN
)));
540 static inline void *alloc_code_gen_buffer(void)
542 void *buf
= static_code_gen_buffer
;
544 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
545 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
548 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
551 #elif defined(USE_MMAP)
552 static inline void *alloc_code_gen_buffer(void)
554 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
558 /* Constrain the position of the buffer based on the host cpu.
559 Note that these addresses are chosen in concert with the
560 addresses assigned in the relevant linker script file. */
561 # if defined(__PIE__) || defined(__PIC__)
562 /* Don't bother setting a preferred location if we're building
563 a position-independent executable. We're more likely to get
564 an address near the main executable if we let the kernel
565 choose the address. */
566 # elif defined(__x86_64__) && defined(MAP_32BIT)
567 /* Force the memory down into low memory with the executable.
568 Leave the choice of exact location with the kernel. */
570 /* Cannot expect to map more than 800MB in low memory. */
571 if (tcg_ctx
.code_gen_buffer_size
> 800u * 1024 * 1024) {
572 tcg_ctx
.code_gen_buffer_size
= 800u * 1024 * 1024;
574 # elif defined(__sparc__)
575 start
= 0x40000000ul
;
576 # elif defined(__s390x__)
577 start
= 0x90000000ul
;
578 # elif defined(__mips__)
579 /* ??? We ought to more explicitly manage layout for softmmu too. */
580 # ifdef CONFIG_USER_ONLY
581 start
= 0x68000000ul
;
582 # elif _MIPS_SIM == _ABI64
583 start
= 0x128000000ul
;
585 start
= 0x08000000ul
;
589 buf
= mmap((void *)start
, tcg_ctx
.code_gen_buffer_size
,
590 PROT_WRITE
| PROT_READ
| PROT_EXEC
, flags
, -1, 0);
591 if (buf
== MAP_FAILED
) {
596 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
597 /* Try again, with the original still mapped, to avoid re-acquiring
598 that 256mb crossing. This time don't specify an address. */
599 size_t size2
, size1
= tcg_ctx
.code_gen_buffer_size
;
600 void *buf2
= mmap(NULL
, size1
, PROT_WRITE
| PROT_READ
| PROT_EXEC
,
602 if (buf2
!= MAP_FAILED
) {
603 if (!cross_256mb(buf2
, size1
)) {
604 /* Success! Use the new buffer. */
608 /* Failure. Work with what we had. */
612 /* Split the original buffer. Free the smaller half. */
613 buf2
= split_cross_256mb(buf
, size1
);
614 size2
= tcg_ctx
.code_gen_buffer_size
;
615 munmap(buf
+ (buf
== buf2
? size2
: 0), size1
- size2
);
623 static inline void *alloc_code_gen_buffer(void)
625 void *buf
= g_malloc(tcg_ctx
.code_gen_buffer_size
);
632 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
633 void *buf2
= g_malloc(tcg_ctx
.code_gen_buffer_size
);
634 if (buf2
!= NULL
&& !cross_256mb(buf2
, size1
)) {
635 /* Success! Use the new buffer. */
639 /* Failure. Work with what we had. Since this is malloc
640 and not mmap, we can't free the other half. */
642 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
647 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
650 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
652 static inline void code_gen_alloc(size_t tb_size
)
654 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
655 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
656 if (tcg_ctx
.code_gen_buffer
== NULL
) {
657 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
661 qemu_madvise(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
,
664 /* Steal room for the prologue at the end of the buffer. This ensures
665 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
666 from TB's to the prologue are going to be in range. It also means
667 that we don't need to mark (additional) portions of the data segment
669 tcg_ctx
.code_gen_prologue
= tcg_ctx
.code_gen_buffer
+
670 tcg_ctx
.code_gen_buffer_size
- 1024;
671 tcg_ctx
.code_gen_buffer_size
-= 1024;
673 tcg_ctx
.code_gen_buffer_max_size
= tcg_ctx
.code_gen_buffer_size
-
674 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
675 tcg_ctx
.code_gen_max_blocks
= tcg_ctx
.code_gen_buffer_size
/
676 CODE_GEN_AVG_BLOCK_SIZE
;
678 g_malloc(tcg_ctx
.code_gen_max_blocks
* sizeof(TranslationBlock
));
681 /* Must be called before using the QEMU cpus. 'tb_size' is the size
682 (in bytes) allocated to the translation buffer. Zero means default
684 void tcg_exec_init(uintptr_t tb_size
)
687 code_gen_alloc(tb_size
);
688 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
689 tcg_register_jit(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
);
691 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
692 /* There's no guest base to take into account, so go ahead and
693 initialize the prologue now. */
694 tcg_prologue_init(&tcg_ctx
);
698 bool tcg_enabled(void)
700 return tcg_ctx
.code_gen_buffer
!= NULL
;
703 /* Allocate a new translation block. Flush the translation buffer if
704 too many translation blocks or too much generated code. */
705 static TranslationBlock
*tb_alloc(target_ulong pc
)
707 TranslationBlock
*tb
;
709 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
||
710 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) >=
711 tcg_ctx
.code_gen_buffer_max_size
) {
714 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
720 void tb_free(TranslationBlock
*tb
)
722 /* In practice this is mostly used for single use temporary TB
723 Ignore the hard cases and just back up if this TB happens to
724 be the last one generated. */
725 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
726 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
727 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
728 tcg_ctx
.tb_ctx
.nb_tbs
--;
732 static inline void invalidate_page_bitmap(PageDesc
*p
)
734 if (p
->code_bitmap
) {
735 g_free(p
->code_bitmap
);
736 p
->code_bitmap
= NULL
;
738 p
->code_write_count
= 0;
741 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
742 static void page_flush_tb_1(int level
, void **lp
)
752 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
753 pd
[i
].first_tb
= NULL
;
754 invalidate_page_bitmap(pd
+ i
);
759 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
760 page_flush_tb_1(level
- 1, pp
+ i
);
765 static void page_flush_tb(void)
769 for (i
= 0; i
< V_L1_SIZE
; i
++) {
770 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
774 /* flush all the translation blocks */
775 /* XXX: tb_flush is currently not thread safe */
776 void tb_flush(CPUArchState
*env1
)
778 CPUState
*cpu
= ENV_GET_CPU(env1
);
780 #if defined(DEBUG_FLUSH)
781 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
782 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
783 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
784 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
785 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
787 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
788 > tcg_ctx
.code_gen_buffer_size
) {
789 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
791 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
794 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
797 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
800 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
801 /* XXX: flush processor icache at this point if cache flush is
803 tcg_ctx
.tb_ctx
.tb_flush_count
++;
806 #ifdef DEBUG_TB_CHECK
808 static void tb_invalidate_check(target_ulong address
)
810 TranslationBlock
*tb
;
813 address
&= TARGET_PAGE_MASK
;
814 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
815 for (tb
= tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
816 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
817 address
>= tb
->pc
+ tb
->size
)) {
818 printf("ERROR invalidate: address=" TARGET_FMT_lx
819 " PC=%08lx size=%04x\n",
820 address
, (long)tb
->pc
, tb
->size
);
826 /* verify that all the pages have correct rights for code */
827 static void tb_page_check(void)
829 TranslationBlock
*tb
;
830 int i
, flags1
, flags2
;
832 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
833 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
834 tb
= tb
->phys_hash_next
) {
835 flags1
= page_get_flags(tb
->pc
);
836 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
837 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
838 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
839 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
847 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
849 TranslationBlock
*tb1
;
854 *ptb
= tb1
->phys_hash_next
;
857 ptb
= &tb1
->phys_hash_next
;
861 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
863 TranslationBlock
*tb1
;
868 n1
= (uintptr_t)tb1
& 3;
869 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
871 *ptb
= tb1
->page_next
[n1
];
874 ptb
= &tb1
->page_next
[n1
];
878 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
880 TranslationBlock
*tb1
, **ptb
;
883 ptb
= &tb
->jmp_next
[n
];
886 /* find tb(n) in circular list */
889 n1
= (uintptr_t)tb1
& 3;
890 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
891 if (n1
== n
&& tb1
== tb
) {
895 ptb
= &tb1
->jmp_first
;
897 ptb
= &tb1
->jmp_next
[n1
];
900 /* now we can suppress tb(n) from the list */
901 *ptb
= tb
->jmp_next
[n
];
903 tb
->jmp_next
[n
] = NULL
;
907 /* reset the jump entry 'n' of a TB so that it is not chained to
909 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
911 tb_set_jmp_target(tb
, n
, (uintptr_t)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
914 /* invalidate one TB */
915 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
920 tb_page_addr_t phys_pc
;
921 TranslationBlock
*tb1
, *tb2
;
923 /* remove the TB from the hash list */
924 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
925 h
= tb_phys_hash_func(phys_pc
);
926 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
928 /* remove the TB from the page list */
929 if (tb
->page_addr
[0] != page_addr
) {
930 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
931 tb_page_remove(&p
->first_tb
, tb
);
932 invalidate_page_bitmap(p
);
934 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
935 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
936 tb_page_remove(&p
->first_tb
, tb
);
937 invalidate_page_bitmap(p
);
940 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
942 /* remove the TB from the hash list */
943 h
= tb_jmp_cache_hash_func(tb
->pc
);
945 if (cpu
->tb_jmp_cache
[h
] == tb
) {
946 cpu
->tb_jmp_cache
[h
] = NULL
;
950 /* suppress this TB from the two jump lists */
951 tb_jmp_remove(tb
, 0);
952 tb_jmp_remove(tb
, 1);
954 /* suppress any remaining jumps to this TB */
957 n1
= (uintptr_t)tb1
& 3;
961 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
962 tb2
= tb1
->jmp_next
[n1
];
963 tb_reset_jump(tb1
, n1
);
964 tb1
->jmp_next
[n1
] = NULL
;
967 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2); /* fail safe */
969 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
972 static inline void set_bits(uint8_t *tab
, int start
, int len
)
978 mask
= 0xff << (start
& 7);
979 if ((start
& ~7) == (end
& ~7)) {
981 mask
&= ~(0xff << (end
& 7));
986 start
= (start
+ 8) & ~7;
988 while (start
< end1
) {
993 mask
= ~(0xff << (end
& 7));
999 static void build_page_bitmap(PageDesc
*p
)
1001 int n
, tb_start
, tb_end
;
1002 TranslationBlock
*tb
;
1004 p
->code_bitmap
= g_malloc0(TARGET_PAGE_SIZE
/ 8);
1007 while (tb
!= NULL
) {
1008 n
= (uintptr_t)tb
& 3;
1009 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1010 /* NOTE: this is subtle as a TB may span two physical pages */
1012 /* NOTE: tb_end may be after the end of the page, but
1013 it is not a problem */
1014 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1015 tb_end
= tb_start
+ tb
->size
;
1016 if (tb_end
> TARGET_PAGE_SIZE
) {
1017 tb_end
= TARGET_PAGE_SIZE
;
1021 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1023 set_bits(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1024 tb
= tb
->page_next
[n
];
1028 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1029 target_ulong pc
, target_ulong cs_base
,
1030 int flags
, int cflags
)
1032 CPUArchState
*env
= cpu
->env_ptr
;
1033 TranslationBlock
*tb
;
1034 tb_page_addr_t phys_pc
, phys_page2
;
1035 target_ulong virt_page2
;
1038 phys_pc
= get_page_addr_code(env
, pc
);
1041 /* flush must be done */
1043 /* cannot fail at this point */
1045 /* Don't forget to invalidate previous TB info. */
1046 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
1048 tb
->tc_ptr
= tcg_ctx
.code_gen_ptr
;
1049 tb
->cs_base
= cs_base
;
1051 tb
->cflags
= cflags
;
1052 cpu_gen_code(env
, tb
, &code_gen_size
);
1053 tcg_ctx
.code_gen_ptr
= (void *)(((uintptr_t)tcg_ctx
.code_gen_ptr
+
1054 code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1056 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1057 /* if we are doing vsyscall don't link the page as it lies in high memory
1058 and tb_alloc_page will abort due to page_l1_map returning NULL */
1059 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1060 && phys_pc
< TARGET_VSYSCALL_END
))
1064 /* check next page if needed */
1065 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1067 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1068 phys_page2
= get_page_addr_code(env
, virt_page2
);
1070 tb_link_page(tb
, phys_pc
, phys_page2
);
1075 * Invalidate all TBs which intersect with the target physical address range
1076 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1077 * 'is_cpu_write_access' should be true if called from a real cpu write
1078 * access: the virtual CPU will exit the current TB if code is modified inside
1081 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
,
1082 int is_cpu_write_access
)
1084 while (start
< end
) {
1085 tb_invalidate_phys_page_range(start
, end
, is_cpu_write_access
);
1086 start
&= TARGET_PAGE_MASK
;
1087 start
+= TARGET_PAGE_SIZE
;
1092 * Invalidate all TBs which intersect with the target physical address range
1093 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1094 * 'is_cpu_write_access' should be true if called from a real cpu write
1095 * access: the virtual CPU will exit the current TB if code is modified inside
1098 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1099 int is_cpu_write_access
)
1101 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1102 CPUState
*cpu
= current_cpu
;
1103 #if defined(TARGET_HAS_PRECISE_SMC)
1104 CPUArchState
*env
= NULL
;
1106 tb_page_addr_t tb_start
, tb_end
;
1109 #ifdef TARGET_HAS_PRECISE_SMC
1110 int current_tb_not_found
= is_cpu_write_access
;
1111 TranslationBlock
*current_tb
= NULL
;
1112 int current_tb_modified
= 0;
1113 target_ulong current_pc
= 0;
1114 target_ulong current_cs_base
= 0;
1115 int current_flags
= 0;
1116 #endif /* TARGET_HAS_PRECISE_SMC */
1118 p
= page_find(start
>> TARGET_PAGE_BITS
);
1122 if (!p
->code_bitmap
&&
1123 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
&&
1124 is_cpu_write_access
) {
1125 /* build code bitmap */
1126 build_page_bitmap(p
);
1128 #if defined(TARGET_HAS_PRECISE_SMC)
1134 /* we remove all the TBs in the range [start, end[ */
1135 /* XXX: see if in some cases it could be faster to invalidate all
1138 while (tb
!= NULL
) {
1139 n
= (uintptr_t)tb
& 3;
1140 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1141 tb_next
= tb
->page_next
[n
];
1142 /* NOTE: this is subtle as a TB may span two physical pages */
1144 /* NOTE: tb_end may be after the end of the page, but
1145 it is not a problem */
1146 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1147 tb_end
= tb_start
+ tb
->size
;
1149 tb_start
= tb
->page_addr
[1];
1150 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1152 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1153 #ifdef TARGET_HAS_PRECISE_SMC
1154 if (current_tb_not_found
) {
1155 current_tb_not_found
= 0;
1157 if (cpu
->mem_io_pc
) {
1158 /* now we have a real cpu fault */
1159 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1162 if (current_tb
== tb
&&
1163 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1164 /* If we are modifying the current TB, we must stop
1165 its execution. We could be more precise by checking
1166 that the modification is after the current PC, but it
1167 would require a specialized function to partially
1168 restore the CPU state */
1170 current_tb_modified
= 1;
1171 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1172 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1175 #endif /* TARGET_HAS_PRECISE_SMC */
1176 /* we need to do that to handle the case where a signal
1177 occurs while doing tb_phys_invalidate() */
1180 saved_tb
= cpu
->current_tb
;
1181 cpu
->current_tb
= NULL
;
1183 tb_phys_invalidate(tb
, -1);
1185 cpu
->current_tb
= saved_tb
;
1186 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1187 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1193 #if !defined(CONFIG_USER_ONLY)
1194 /* if no code remaining, no need to continue to use slow writes */
1196 invalidate_page_bitmap(p
);
1197 if (is_cpu_write_access
) {
1198 tlb_unprotect_code_phys(cpu
, start
, cpu
->mem_io_vaddr
);
1202 #ifdef TARGET_HAS_PRECISE_SMC
1203 if (current_tb_modified
) {
1204 /* we generate a block containing just the instruction
1205 modifying the memory. It will ensure that it cannot modify
1207 cpu
->current_tb
= NULL
;
1208 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1209 cpu_resume_from_signal(cpu
, NULL
);
1214 /* len must be <= 8 and start must be a multiple of len */
1215 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1222 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1223 cpu_single_env
->mem_io_vaddr
, len
,
1224 cpu_single_env
->eip
,
1225 cpu_single_env
->eip
+
1226 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1229 p
= page_find(start
>> TARGET_PAGE_BITS
);
1233 if (p
->code_bitmap
) {
1234 offset
= start
& ~TARGET_PAGE_MASK
;
1235 b
= p
->code_bitmap
[offset
>> 3] >> (offset
& 7);
1236 if (b
& ((1 << len
) - 1)) {
1241 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1245 #if !defined(CONFIG_SOFTMMU)
1246 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1247 uintptr_t pc
, void *puc
,
1250 TranslationBlock
*tb
;
1253 #ifdef TARGET_HAS_PRECISE_SMC
1254 TranslationBlock
*current_tb
= NULL
;
1255 CPUState
*cpu
= current_cpu
;
1256 CPUArchState
*env
= NULL
;
1257 int current_tb_modified
= 0;
1258 target_ulong current_pc
= 0;
1259 target_ulong current_cs_base
= 0;
1260 int current_flags
= 0;
1263 addr
&= TARGET_PAGE_MASK
;
1264 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1269 #ifdef TARGET_HAS_PRECISE_SMC
1270 if (tb
&& pc
!= 0) {
1271 current_tb
= tb_find_pc(pc
);
1277 while (tb
!= NULL
) {
1278 n
= (uintptr_t)tb
& 3;
1279 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1280 #ifdef TARGET_HAS_PRECISE_SMC
1281 if (current_tb
== tb
&&
1282 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1283 /* If we are modifying the current TB, we must stop
1284 its execution. We could be more precise by checking
1285 that the modification is after the current PC, but it
1286 would require a specialized function to partially
1287 restore the CPU state */
1289 current_tb_modified
= 1;
1290 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1291 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1294 #endif /* TARGET_HAS_PRECISE_SMC */
1295 tb_phys_invalidate(tb
, addr
);
1296 tb
= tb
->page_next
[n
];
1299 #ifdef TARGET_HAS_PRECISE_SMC
1300 if (current_tb_modified
) {
1301 /* we generate a block containing just the instruction
1302 modifying the memory. It will ensure that it cannot modify
1304 cpu
->current_tb
= NULL
;
1305 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1309 cpu_resume_from_signal(cpu
, puc
);
1315 /* add the tb in the target page and protect it if necessary */
1316 static inline void tb_alloc_page(TranslationBlock
*tb
,
1317 unsigned int n
, tb_page_addr_t page_addr
)
1320 #ifndef CONFIG_USER_ONLY
1321 bool page_already_protected
;
1324 tb
->page_addr
[n
] = page_addr
;
1325 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1326 tb
->page_next
[n
] = p
->first_tb
;
1327 #ifndef CONFIG_USER_ONLY
1328 page_already_protected
= p
->first_tb
!= NULL
;
1330 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1331 invalidate_page_bitmap(p
);
1333 #if defined(TARGET_HAS_SMC) || 1
1335 #if defined(CONFIG_USER_ONLY)
1336 if (p
->flags
& PAGE_WRITE
) {
1341 /* force the host page as non writable (writes will have a
1342 page fault + mprotect overhead) */
1343 page_addr
&= qemu_host_page_mask
;
1345 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1346 addr
+= TARGET_PAGE_SIZE
) {
1348 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1353 p2
->flags
&= ~PAGE_WRITE
;
1355 mprotect(g2h(page_addr
), qemu_host_page_size
,
1356 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1357 #ifdef DEBUG_TB_INVALIDATE
1358 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1363 /* if some code is already present, then the pages are already
1364 protected. So we handle the case where only the first TB is
1365 allocated in a physical page */
1366 if (!page_already_protected
) {
1367 tlb_protect_code(page_addr
);
1371 #endif /* TARGET_HAS_SMC */
1374 /* add a new TB and link it to the physical page tables. phys_page2 is
1375 (-1) to indicate that only one page contains the TB. */
1376 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1377 tb_page_addr_t phys_page2
)
1380 TranslationBlock
**ptb
;
1382 /* Grab the mmap lock to stop another thread invalidating this TB
1383 before we are done. */
1385 /* add in the physical hash table */
1386 h
= tb_phys_hash_func(phys_pc
);
1387 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1388 tb
->phys_hash_next
= *ptb
;
1391 /* add in the page list */
1392 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1393 if (phys_page2
!= -1) {
1394 tb_alloc_page(tb
, 1, phys_page2
);
1396 tb
->page_addr
[1] = -1;
1399 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2);
1400 tb
->jmp_next
[0] = NULL
;
1401 tb
->jmp_next
[1] = NULL
;
1403 /* init original jump addresses */
1404 if (tb
->tb_next_offset
[0] != 0xffff) {
1405 tb_reset_jump(tb
, 0);
1407 if (tb
->tb_next_offset
[1] != 0xffff) {
1408 tb_reset_jump(tb
, 1);
1411 #ifdef DEBUG_TB_CHECK
1417 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1418 tb[1].tc_ptr. Return NULL if not found */
1419 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1421 int m_min
, m_max
, m
;
1423 TranslationBlock
*tb
;
1425 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1428 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1429 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1432 /* binary search (cf Knuth) */
1434 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1435 while (m_min
<= m_max
) {
1436 m
= (m_min
+ m_max
) >> 1;
1437 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1438 v
= (uintptr_t)tb
->tc_ptr
;
1441 } else if (tc_ptr
< v
) {
1447 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1450 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1451 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1453 ram_addr_t ram_addr
;
1457 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1458 if (!(memory_region_is_ram(mr
)
1459 || memory_region_is_romd(mr
))) {
1462 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1464 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1466 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1468 void tb_check_watchpoint(CPUState
*cpu
)
1470 TranslationBlock
*tb
;
1472 tb
= tb_find_pc(cpu
->mem_io_pc
);
1474 cpu_abort(cpu
, "check_watchpoint: could not find TB for pc=%p",
1475 (void *)cpu
->mem_io_pc
);
1477 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1478 tb_phys_invalidate(tb
, -1);
1481 #ifndef CONFIG_USER_ONLY
1482 /* mask must never be zero, except for A20 change call */
1483 static void tcg_handle_interrupt(CPUState
*cpu
, int mask
)
1487 old_mask
= cpu
->interrupt_request
;
1488 cpu
->interrupt_request
|= mask
;
1491 * If called from iothread context, wake the target cpu in
1494 if (!qemu_cpu_is_self(cpu
)) {
1500 cpu
->icount_decr
.u16
.high
= 0xffff;
1501 if (!cpu_can_do_io(cpu
)
1502 && (mask
& ~old_mask
) != 0) {
1503 cpu_abort(cpu
, "Raised interrupt while not in I/O function");
1506 cpu
->tcg_exit_req
= 1;
1510 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1512 /* in deterministic execution mode, instructions doing device I/Os
1513 must be at the end of the TB */
1514 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1516 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1517 CPUArchState
*env
= cpu
->env_ptr
;
1519 TranslationBlock
*tb
;
1521 target_ulong pc
, cs_base
;
1524 tb
= tb_find_pc(retaddr
);
1526 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1529 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1530 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1531 /* Calculate how many instructions had been executed before the fault
1533 n
= n
- cpu
->icount_decr
.u16
.low
;
1534 /* Generate a new TB ending on the I/O insn. */
1536 /* On MIPS and SH, delay slot instructions can only be restarted if
1537 they were already the first instruction in the TB. If this is not
1538 the first instruction in a TB then re-execute the preceding
1540 #if defined(TARGET_MIPS)
1541 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1542 env
->active_tc
.PC
-= 4;
1543 cpu
->icount_decr
.u16
.low
++;
1544 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1546 #elif defined(TARGET_SH4)
1547 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1550 cpu
->icount_decr
.u16
.low
++;
1551 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1554 /* This should never happen. */
1555 if (n
> CF_COUNT_MASK
) {
1556 cpu_abort(cpu
, "TB too big during recompile");
1559 cflags
= n
| CF_LAST_IO
;
1561 cs_base
= tb
->cs_base
;
1563 tb_phys_invalidate(tb
, -1);
1564 /* FIXME: In theory this could raise an exception. In practice
1565 we have already translated the block once so it's probably ok. */
1566 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1567 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1568 the first in the TB) then we end up generating a whole new TB and
1569 repeating the fault, which is horribly inefficient.
1570 Better would be to execute just this insn uncached, or generate a
1572 cpu_resume_from_signal(cpu
, NULL
);
1575 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1579 /* Discard jump cache entries for any tb which might potentially
1580 overlap the flushed page. */
1581 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1582 memset(&cpu
->tb_jmp_cache
[i
], 0,
1583 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1585 i
= tb_jmp_cache_hash_page(addr
);
1586 memset(&cpu
->tb_jmp_cache
[i
], 0,
1587 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1590 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1592 int i
, target_code_size
, max_target_code_size
;
1593 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1594 TranslationBlock
*tb
;
1596 target_code_size
= 0;
1597 max_target_code_size
= 0;
1599 direct_jmp_count
= 0;
1600 direct_jmp2_count
= 0;
1601 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1602 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1603 target_code_size
+= tb
->size
;
1604 if (tb
->size
> max_target_code_size
) {
1605 max_target_code_size
= tb
->size
;
1607 if (tb
->page_addr
[1] != -1) {
1610 if (tb
->tb_next_offset
[0] != 0xffff) {
1612 if (tb
->tb_next_offset
[1] != 0xffff) {
1613 direct_jmp2_count
++;
1617 /* XXX: avoid using doubles ? */
1618 cpu_fprintf(f
, "Translation buffer state:\n");
1619 cpu_fprintf(f
, "gen code size %td/%zd\n",
1620 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1621 tcg_ctx
.code_gen_buffer_max_size
);
1622 cpu_fprintf(f
, "TB count %d/%d\n",
1623 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1624 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1625 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1626 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1627 max_target_code_size
);
1628 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1629 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1630 tcg_ctx
.code_gen_buffer
) /
1631 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1632 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1633 tcg_ctx
.code_gen_buffer
) /
1634 target_code_size
: 0);
1635 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1636 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1637 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1638 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1640 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1641 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1643 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1644 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1645 cpu_fprintf(f
, "\nStatistics:\n");
1646 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1647 cpu_fprintf(f
, "TB invalidate count %d\n",
1648 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1649 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1650 tcg_dump_info(f
, cpu_fprintf
);
1653 #else /* CONFIG_USER_ONLY */
1655 void cpu_interrupt(CPUState
*cpu
, int mask
)
1657 cpu
->interrupt_request
|= mask
;
1658 cpu
->tcg_exit_req
= 1;
1662 * Walks guest process memory "regions" one by one
1663 * and calls callback function 'fn' for each region.
1665 struct walk_memory_regions_data
{
1666 walk_memory_regions_fn fn
;
1672 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1673 abi_ulong end
, int new_prot
)
1675 if (data
->start
!= -1ul) {
1676 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1682 data
->start
= (new_prot
? end
: -1ul);
1683 data
->prot
= new_prot
;
1688 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1689 abi_ulong base
, int level
, void **lp
)
1695 return walk_memory_regions_end(data
, base
, 0);
1701 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1702 int prot
= pd
[i
].flags
;
1704 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1705 if (prot
!= data
->prot
) {
1706 rc
= walk_memory_regions_end(data
, pa
, prot
);
1715 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1716 pa
= base
| ((abi_ulong
)i
<<
1717 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1718 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1728 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1730 struct walk_memory_regions_data data
;
1738 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1739 int rc
= walk_memory_regions_1(&data
, (abi_ulong
)i
<< V_L1_SHIFT
,
1740 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1747 return walk_memory_regions_end(&data
, 0, 0);
1750 static int dump_region(void *priv
, abi_ulong start
,
1751 abi_ulong end
, abi_ulong prot
)
1753 FILE *f
= (FILE *)priv
;
1755 (void) fprintf(f
, TARGET_ABI_FMT_lx
"-"TARGET_ABI_FMT_lx
1756 " "TARGET_ABI_FMT_lx
" %c%c%c\n",
1757 start
, end
, end
- start
,
1758 ((prot
& PAGE_READ
) ? 'r' : '-'),
1759 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1760 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1765 /* dump memory mappings */
1766 void page_dump(FILE *f
)
1768 const int length
= sizeof(abi_ulong
) * 2;
1769 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1770 length
, "start", length
, "end", length
, "size", "prot");
1771 walk_memory_regions(f
, dump_region
);
1774 int page_get_flags(target_ulong address
)
1778 p
= page_find(address
>> TARGET_PAGE_BITS
);
1785 /* Modify the flags of a page and invalidate the code if necessary.
1786 The flag PAGE_WRITE_ORG is positioned automatically depending
1787 on PAGE_WRITE. The mmap_lock should already be held. */
1788 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1790 target_ulong addr
, len
;
1792 /* This function should never be called with addresses outside the
1793 guest address space. If this assert fires, it probably indicates
1794 a missing call to h2g_valid. */
1795 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1796 assert(end
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1798 assert(start
< end
);
1800 start
= start
& TARGET_PAGE_MASK
;
1801 end
= TARGET_PAGE_ALIGN(end
);
1803 if (flags
& PAGE_WRITE
) {
1804 flags
|= PAGE_WRITE_ORG
;
1807 for (addr
= start
, len
= end
- start
;
1809 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1810 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1812 /* If the write protection bit is set, then we invalidate
1814 if (!(p
->flags
& PAGE_WRITE
) &&
1815 (flags
& PAGE_WRITE
) &&
1817 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1823 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1829 /* This function should never be called with addresses outside the
1830 guest address space. If this assert fires, it probably indicates
1831 a missing call to h2g_valid. */
1832 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1833 assert(start
< ((abi_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1839 if (start
+ len
- 1 < start
) {
1840 /* We've wrapped around. */
1844 /* must do before we loose bits in the next step */
1845 end
= TARGET_PAGE_ALIGN(start
+ len
);
1846 start
= start
& TARGET_PAGE_MASK
;
1848 for (addr
= start
, len
= end
- start
;
1850 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1851 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1855 if (!(p
->flags
& PAGE_VALID
)) {
1859 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1862 if (flags
& PAGE_WRITE
) {
1863 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1866 /* unprotect the page if it was put read-only because it
1867 contains translated code */
1868 if (!(p
->flags
& PAGE_WRITE
)) {
1869 if (!page_unprotect(addr
, 0, NULL
)) {
1878 /* called from signal handler: invalidate the code and unprotect the
1879 page. Return TRUE if the fault was successfully handled. */
1880 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1884 target_ulong host_start
, host_end
, addr
;
1886 /* Technically this isn't safe inside a signal handler. However we
1887 know this only ever happens in a synchronous SEGV handler, so in
1888 practice it seems to be ok. */
1891 p
= page_find(address
>> TARGET_PAGE_BITS
);
1897 /* if the page was really writable, then we change its
1898 protection back to writable */
1899 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
1900 host_start
= address
& qemu_host_page_mask
;
1901 host_end
= host_start
+ qemu_host_page_size
;
1904 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
1905 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1906 p
->flags
|= PAGE_WRITE
;
1909 /* and since the content will be modified, we must invalidate
1910 the corresponding translated code. */
1911 tb_invalidate_phys_page(addr
, pc
, puc
, true);
1912 #ifdef DEBUG_TB_CHECK
1913 tb_invalidate_check(addr
);
1916 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
1925 #endif /* CONFIG_USER_ONLY */