4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
23 #include <sys/types.h>
27 #include "qemu-common.h"
30 #include "qemu-common.h"
31 #define NO_CPU_IO_DEFS
34 #include "disas/disas.h"
36 #if defined(CONFIG_USER_ONLY)
38 #if defined(TARGET_X86_64)
41 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42 #include <sys/param.h>
43 #if __FreeBSD_version >= 700104
44 #define HAVE_KINFO_GETVMMAP
45 #define sigqueue sigqueue_freebsd /* avoid redefinition */
48 #include <machine/profile.h>
57 #include "exec/address-spaces.h"
60 #include "exec/cputlb.h"
61 #include "exec/tb-hash.h"
62 #include "translate-all.h"
63 #include "qemu/bitmap.h"
64 #include "qemu/timer.h"
66 //#define DEBUG_TB_INVALIDATE
68 /* make various TB consistency checks */
69 //#define DEBUG_TB_CHECK
71 #if !defined(CONFIG_USER_ONLY)
72 /* TB consistency checks only implemented for usermode emulation. */
76 #define SMC_BITMAP_USE_THRESHOLD 10
78 typedef struct PageDesc
{
79 /* list of TBs intersecting this ram page */
80 TranslationBlock
*first_tb
;
81 /* in order to optimize self modifying code, we count the number
82 of lookups we do to a given page to use a bitmap */
83 unsigned int code_write_count
;
84 unsigned long *code_bitmap
;
85 #if defined(CONFIG_USER_ONLY)
90 /* In system mode we want L1_MAP to be based on ram offsets,
91 while in user mode we want it to be based on virtual addresses. */
92 #if !defined(CONFIG_USER_ONLY)
93 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
94 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
96 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
99 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
102 /* Size of the L2 (and L3, etc) page tables. */
104 #define V_L2_SIZE (1 << V_L2_BITS)
106 /* The bits remaining after N lower levels of page tables. */
107 #define V_L1_BITS_REM \
108 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
110 #if V_L1_BITS_REM < 4
111 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
113 #define V_L1_BITS V_L1_BITS_REM
116 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
118 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
120 uintptr_t qemu_real_host_page_size
;
121 uintptr_t qemu_real_host_page_mask
;
122 uintptr_t qemu_host_page_size
;
123 uintptr_t qemu_host_page_mask
;
125 /* This is a multi-level map on the virtual address space.
126 The bottom level has pointers to PageDesc. */
127 static void *l1_map
[V_L1_SIZE
];
129 /* code generation context */
132 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
133 tb_page_addr_t phys_page2
);
134 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
136 void cpu_gen_init(void)
138 tcg_context_init(&tcg_ctx
);
141 /* return non zero if the very first instruction is invalid so that
142 the virtual CPU can trigger an exception.
144 '*gen_code_size_ptr' contains the size of the generated code (host
147 int cpu_gen_code(CPUArchState
*env
, TranslationBlock
*tb
, int *gen_code_size_ptr
)
149 TCGContext
*s
= &tcg_ctx
;
150 tcg_insn_unit
*gen_code_buf
;
152 #ifdef CONFIG_PROFILER
156 #ifdef CONFIG_PROFILER
157 s
->tb_count1
++; /* includes aborted translations because of
159 ti
= profile_getclock();
163 gen_intermediate_code(env
, tb
);
165 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
167 /* generate machine code */
168 gen_code_buf
= tb
->tc_ptr
;
169 tb
->tb_next_offset
[0] = 0xffff;
170 tb
->tb_next_offset
[1] = 0xffff;
171 s
->tb_next_offset
= tb
->tb_next_offset
;
172 #ifdef USE_DIRECT_JUMP
173 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
176 s
->tb_jmp_offset
= NULL
;
177 s
->tb_next
= tb
->tb_next
;
180 #ifdef CONFIG_PROFILER
182 s
->interm_time
+= profile_getclock() - ti
;
183 s
->code_time
-= profile_getclock();
185 gen_code_size
= tcg_gen_code(s
, gen_code_buf
);
186 *gen_code_size_ptr
= gen_code_size
;
187 #ifdef CONFIG_PROFILER
188 s
->code_time
+= profile_getclock();
189 s
->code_in_len
+= tb
->size
;
190 s
->code_out_len
+= gen_code_size
;
194 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
)) {
195 qemu_log("OUT: [size=%d]\n", gen_code_size
);
196 log_disas(tb
->tc_ptr
, gen_code_size
);
204 /* The cpu state corresponding to 'searched_pc' is restored.
206 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
207 uintptr_t searched_pc
)
209 CPUArchState
*env
= cpu
->env_ptr
;
210 TCGContext
*s
= &tcg_ctx
;
213 #ifdef CONFIG_PROFILER
217 #ifdef CONFIG_PROFILER
218 ti
= profile_getclock();
222 gen_intermediate_code_pc(env
, tb
);
224 if (tb
->cflags
& CF_USE_ICOUNT
) {
226 /* Reset the cycle counter to the start of the block. */
227 cpu
->icount_decr
.u16
.low
+= tb
->icount
;
228 /* Clear the IO flag. */
232 /* find opc index corresponding to search_pc */
233 tc_ptr
= (uintptr_t)tb
->tc_ptr
;
234 if (searched_pc
< tc_ptr
)
237 s
->tb_next_offset
= tb
->tb_next_offset
;
238 #ifdef USE_DIRECT_JUMP
239 s
->tb_jmp_offset
= tb
->tb_jmp_offset
;
242 s
->tb_jmp_offset
= NULL
;
243 s
->tb_next
= tb
->tb_next
;
245 j
= tcg_gen_code_search_pc(s
, (tcg_insn_unit
*)tc_ptr
,
246 searched_pc
- tc_ptr
);
249 /* now find start of instruction before */
250 while (s
->gen_opc_instr_start
[j
] == 0) {
253 cpu
->icount_decr
.u16
.low
-= s
->gen_opc_icount
[j
];
255 restore_state_to_opc(env
, tb
, j
);
257 #ifdef CONFIG_PROFILER
258 s
->restore_time
+= profile_getclock() - ti
;
264 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
266 TranslationBlock
*tb
;
268 tb
= tb_find_pc(retaddr
);
270 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
271 if (tb
->cflags
& CF_NOCACHE
) {
272 /* one-shot translation, invalidate it immediately */
273 cpu
->current_tb
= NULL
;
274 tb_phys_invalidate(tb
, -1);
283 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
286 VirtualProtect(addr
, size
,
287 PAGE_EXECUTE_READWRITE
, &old_protect
);
290 static __attribute__((unused
)) void map_exec(void *addr
, long size
)
292 unsigned long start
, end
, page_size
;
294 page_size
= getpagesize();
295 start
= (unsigned long)addr
;
296 start
&= ~(page_size
- 1);
298 end
= (unsigned long)addr
+ size
;
299 end
+= page_size
- 1;
300 end
&= ~(page_size
- 1);
302 mprotect((void *)start
, end
- start
,
303 PROT_READ
| PROT_WRITE
| PROT_EXEC
);
307 void page_size_init(void)
309 /* NOTE: we can always suppose that qemu_host_page_size >=
311 qemu_real_host_page_size
= getpagesize();
312 qemu_real_host_page_mask
= ~(qemu_real_host_page_size
- 1);
313 if (qemu_host_page_size
== 0) {
314 qemu_host_page_size
= qemu_real_host_page_size
;
316 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
317 qemu_host_page_size
= TARGET_PAGE_SIZE
;
319 qemu_host_page_mask
= ~(qemu_host_page_size
- 1);
322 static void page_init(void)
325 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
327 #ifdef HAVE_KINFO_GETVMMAP
328 struct kinfo_vmentry
*freep
;
331 freep
= kinfo_getvmmap(getpid(), &cnt
);
334 for (i
= 0; i
< cnt
; i
++) {
335 unsigned long startaddr
, endaddr
;
337 startaddr
= freep
[i
].kve_start
;
338 endaddr
= freep
[i
].kve_end
;
339 if (h2g_valid(startaddr
)) {
340 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
342 if (h2g_valid(endaddr
)) {
343 endaddr
= h2g(endaddr
);
344 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
346 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
348 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
359 last_brk
= (unsigned long)sbrk(0);
361 f
= fopen("/compat/linux/proc/self/maps", "r");
366 unsigned long startaddr
, endaddr
;
369 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
371 if (n
== 2 && h2g_valid(startaddr
)) {
372 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
374 if (h2g_valid(endaddr
)) {
375 endaddr
= h2g(endaddr
);
379 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
391 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
397 /* Level 1. Always allocated. */
398 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
401 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
408 p
= g_new0(void *, V_L2_SIZE
);
412 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
420 pd
= g_new0(PageDesc
, V_L2_SIZE
);
424 return pd
+ (index
& (V_L2_SIZE
- 1));
427 static inline PageDesc
*page_find(tb_page_addr_t index
)
429 return page_find_alloc(index
, 0);
432 #if !defined(CONFIG_USER_ONLY)
433 #define mmap_lock() do { } while (0)
434 #define mmap_unlock() do { } while (0)
437 #if defined(CONFIG_USER_ONLY)
438 /* Currently it is not recommended to allocate big chunks of data in
439 user mode. It will change when a dedicated libc will be used. */
440 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
441 region in which the guest needs to run. Revisit this. */
442 #define USE_STATIC_CODE_GEN_BUFFER
445 /* ??? Should configure for this, not list operating systems here. */
446 #if (defined(__linux__) \
447 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
448 || defined(__DragonFly__) || defined(__OpenBSD__) \
449 || defined(__NetBSD__))
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__aarch64__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
466 #elif defined(__arm__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
468 #elif defined(__s390x__)
469 /* We have a +- 4GB range on the branches; leave some slop. */
470 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
471 #elif defined(__mips__)
472 /* We have a 256MB branch region, but leave room to make sure the
473 main executable is also within that region. */
474 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
476 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
479 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
481 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
482 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
483 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
485 static inline size_t size_code_gen_buffer(size_t tb_size
)
487 /* Size the buffer. */
489 #ifdef USE_STATIC_CODE_GEN_BUFFER
490 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
492 /* ??? Needs adjustments. */
493 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
494 static buffer, we could size this on RESERVED_VA, on the text
495 segment size of the executable, or continue to use the default. */
496 tb_size
= (unsigned long)(ram_size
/ 4);
499 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
500 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
502 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
503 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
505 tcg_ctx
.code_gen_buffer_size
= tb_size
;
510 /* In order to use J and JAL within the code_gen_buffer, we require
511 that the buffer not cross a 256MB boundary. */
512 static inline bool cross_256mb(void *addr
, size_t size
)
514 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & 0xf0000000;
517 /* We weren't able to allocate a buffer without crossing that boundary,
518 so make do with the larger portion of the buffer that doesn't cross.
519 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
520 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
522 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & 0xf0000000);
523 size_t size2
= buf1
+ size1
- buf2
;
531 tcg_ctx
.code_gen_buffer_size
= size1
;
536 #ifdef USE_STATIC_CODE_GEN_BUFFER
537 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
538 __attribute__((aligned(CODE_GEN_ALIGN
)));
540 static inline void *alloc_code_gen_buffer(void)
542 void *buf
= static_code_gen_buffer
;
544 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
545 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
548 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
551 #elif defined(USE_MMAP)
552 static inline void *alloc_code_gen_buffer(void)
554 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
558 /* Constrain the position of the buffer based on the host cpu.
559 Note that these addresses are chosen in concert with the
560 addresses assigned in the relevant linker script file. */
561 # if defined(__PIE__) || defined(__PIC__)
562 /* Don't bother setting a preferred location if we're building
563 a position-independent executable. We're more likely to get
564 an address near the main executable if we let the kernel
565 choose the address. */
566 # elif defined(__x86_64__) && defined(MAP_32BIT)
567 /* Force the memory down into low memory with the executable.
568 Leave the choice of exact location with the kernel. */
570 /* Cannot expect to map more than 800MB in low memory. */
571 if (tcg_ctx
.code_gen_buffer_size
> 800u * 1024 * 1024) {
572 tcg_ctx
.code_gen_buffer_size
= 800u * 1024 * 1024;
574 # elif defined(__sparc__)
575 start
= 0x40000000ul
;
576 # elif defined(__s390x__)
577 start
= 0x90000000ul
;
578 # elif defined(__mips__)
579 /* ??? We ought to more explicitly manage layout for softmmu too. */
580 # ifdef CONFIG_USER_ONLY
581 start
= 0x68000000ul
;
582 # elif _MIPS_SIM == _ABI64
583 start
= 0x128000000ul
;
585 start
= 0x08000000ul
;
589 buf
= mmap((void *)start
, tcg_ctx
.code_gen_buffer_size
,
590 PROT_WRITE
| PROT_READ
| PROT_EXEC
, flags
, -1, 0);
591 if (buf
== MAP_FAILED
) {
596 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
597 /* Try again, with the original still mapped, to avoid re-acquiring
598 that 256mb crossing. This time don't specify an address. */
599 size_t size2
, size1
= tcg_ctx
.code_gen_buffer_size
;
600 void *buf2
= mmap(NULL
, size1
, PROT_WRITE
| PROT_READ
| PROT_EXEC
,
602 if (buf2
!= MAP_FAILED
) {
603 if (!cross_256mb(buf2
, size1
)) {
604 /* Success! Use the new buffer. */
608 /* Failure. Work with what we had. */
612 /* Split the original buffer. Free the smaller half. */
613 buf2
= split_cross_256mb(buf
, size1
);
614 size2
= tcg_ctx
.code_gen_buffer_size
;
615 munmap(buf
+ (buf
== buf2
? size2
: 0), size1
- size2
);
623 static inline void *alloc_code_gen_buffer(void)
625 void *buf
= g_try_malloc(tcg_ctx
.code_gen_buffer_size
);
632 if (cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
)) {
633 void *buf2
= g_malloc(tcg_ctx
.code_gen_buffer_size
);
634 if (buf2
!= NULL
&& !cross_256mb(buf2
, size1
)) {
635 /* Success! Use the new buffer. */
639 /* Failure. Work with what we had. Since this is malloc
640 and not mmap, we can't free the other half. */
642 buf
= split_cross_256mb(buf
, tcg_ctx
.code_gen_buffer_size
);
647 map_exec(buf
, tcg_ctx
.code_gen_buffer_size
);
650 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
652 static inline void code_gen_alloc(size_t tb_size
)
654 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
655 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
656 if (tcg_ctx
.code_gen_buffer
== NULL
) {
657 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
661 qemu_madvise(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
,
664 /* Steal room for the prologue at the end of the buffer. This ensures
665 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
666 from TB's to the prologue are going to be in range. It also means
667 that we don't need to mark (additional) portions of the data segment
669 tcg_ctx
.code_gen_prologue
= tcg_ctx
.code_gen_buffer
+
670 tcg_ctx
.code_gen_buffer_size
- 1024;
671 tcg_ctx
.code_gen_buffer_size
-= 1024;
673 tcg_ctx
.code_gen_buffer_max_size
= tcg_ctx
.code_gen_buffer_size
-
674 (TCG_MAX_OP_SIZE
* OPC_BUF_SIZE
);
675 tcg_ctx
.code_gen_max_blocks
= tcg_ctx
.code_gen_buffer_size
/
676 CODE_GEN_AVG_BLOCK_SIZE
;
678 g_malloc(tcg_ctx
.code_gen_max_blocks
* sizeof(TranslationBlock
));
681 /* Must be called before using the QEMU cpus. 'tb_size' is the size
682 (in bytes) allocated to the translation buffer. Zero means default
684 void tcg_exec_init(uintptr_t tb_size
)
687 code_gen_alloc(tb_size
);
688 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
689 tcg_register_jit(tcg_ctx
.code_gen_buffer
, tcg_ctx
.code_gen_buffer_size
);
691 #if defined(CONFIG_SOFTMMU)
692 /* There's no guest base to take into account, so go ahead and
693 initialize the prologue now. */
694 tcg_prologue_init(&tcg_ctx
);
698 bool tcg_enabled(void)
700 return tcg_ctx
.code_gen_buffer
!= NULL
;
703 /* Allocate a new translation block. Flush the translation buffer if
704 too many translation blocks or too much generated code. */
705 static TranslationBlock
*tb_alloc(target_ulong pc
)
707 TranslationBlock
*tb
;
709 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
||
710 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) >=
711 tcg_ctx
.code_gen_buffer_max_size
) {
714 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
720 void tb_free(TranslationBlock
*tb
)
722 /* In practice this is mostly used for single use temporary TB
723 Ignore the hard cases and just back up if this TB happens to
724 be the last one generated. */
725 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
726 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
727 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
728 tcg_ctx
.tb_ctx
.nb_tbs
--;
732 static inline void invalidate_page_bitmap(PageDesc
*p
)
734 if (p
->code_bitmap
) {
735 g_free(p
->code_bitmap
);
736 p
->code_bitmap
= NULL
;
738 p
->code_write_count
= 0;
741 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
742 static void page_flush_tb_1(int level
, void **lp
)
752 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
753 pd
[i
].first_tb
= NULL
;
754 invalidate_page_bitmap(pd
+ i
);
759 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
760 page_flush_tb_1(level
- 1, pp
+ i
);
765 static void page_flush_tb(void)
769 for (i
= 0; i
< V_L1_SIZE
; i
++) {
770 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
774 /* flush all the translation blocks */
775 /* XXX: tb_flush is currently not thread safe */
776 void tb_flush(CPUState
*cpu
)
778 #if defined(DEBUG_FLUSH)
779 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
780 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
781 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
782 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
783 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
785 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
786 > tcg_ctx
.code_gen_buffer_size
) {
787 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
789 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
792 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
795 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
798 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
799 /* XXX: flush processor icache at this point if cache flush is
801 tcg_ctx
.tb_ctx
.tb_flush_count
++;
804 #ifdef DEBUG_TB_CHECK
806 static void tb_invalidate_check(target_ulong address
)
808 TranslationBlock
*tb
;
811 address
&= TARGET_PAGE_MASK
;
812 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
813 for (tb
= tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
; tb
= tb
->phys_hash_next
) {
814 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
815 address
>= tb
->pc
+ tb
->size
)) {
816 printf("ERROR invalidate: address=" TARGET_FMT_lx
817 " PC=%08lx size=%04x\n",
818 address
, (long)tb
->pc
, tb
->size
);
824 /* verify that all the pages have correct rights for code */
825 static void tb_page_check(void)
827 TranslationBlock
*tb
;
828 int i
, flags1
, flags2
;
830 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
831 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
832 tb
= tb
->phys_hash_next
) {
833 flags1
= page_get_flags(tb
->pc
);
834 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
835 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
836 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
837 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
845 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
847 TranslationBlock
*tb1
;
852 *ptb
= tb1
->phys_hash_next
;
855 ptb
= &tb1
->phys_hash_next
;
859 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
861 TranslationBlock
*tb1
;
866 n1
= (uintptr_t)tb1
& 3;
867 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
869 *ptb
= tb1
->page_next
[n1
];
872 ptb
= &tb1
->page_next
[n1
];
876 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
878 TranslationBlock
*tb1
, **ptb
;
881 ptb
= &tb
->jmp_next
[n
];
884 /* find tb(n) in circular list */
887 n1
= (uintptr_t)tb1
& 3;
888 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
889 if (n1
== n
&& tb1
== tb
) {
893 ptb
= &tb1
->jmp_first
;
895 ptb
= &tb1
->jmp_next
[n1
];
898 /* now we can suppress tb(n) from the list */
899 *ptb
= tb
->jmp_next
[n
];
901 tb
->jmp_next
[n
] = NULL
;
905 /* reset the jump entry 'n' of a TB so that it is not chained to
907 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
909 tb_set_jmp_target(tb
, n
, (uintptr_t)(tb
->tc_ptr
+ tb
->tb_next_offset
[n
]));
912 /* invalidate one TB */
913 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
918 tb_page_addr_t phys_pc
;
919 TranslationBlock
*tb1
, *tb2
;
921 /* remove the TB from the hash list */
922 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
923 h
= tb_phys_hash_func(phys_pc
);
924 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
926 /* remove the TB from the page list */
927 if (tb
->page_addr
[0] != page_addr
) {
928 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
929 tb_page_remove(&p
->first_tb
, tb
);
930 invalidate_page_bitmap(p
);
932 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
933 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
934 tb_page_remove(&p
->first_tb
, tb
);
935 invalidate_page_bitmap(p
);
938 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
940 /* remove the TB from the hash list */
941 h
= tb_jmp_cache_hash_func(tb
->pc
);
943 if (cpu
->tb_jmp_cache
[h
] == tb
) {
944 cpu
->tb_jmp_cache
[h
] = NULL
;
948 /* suppress this TB from the two jump lists */
949 tb_jmp_remove(tb
, 0);
950 tb_jmp_remove(tb
, 1);
952 /* suppress any remaining jumps to this TB */
955 n1
= (uintptr_t)tb1
& 3;
959 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
960 tb2
= tb1
->jmp_next
[n1
];
961 tb_reset_jump(tb1
, n1
);
962 tb1
->jmp_next
[n1
] = NULL
;
965 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2); /* fail safe */
967 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
970 static void build_page_bitmap(PageDesc
*p
)
972 int n
, tb_start
, tb_end
;
973 TranslationBlock
*tb
;
975 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
979 n
= (uintptr_t)tb
& 3;
980 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
981 /* NOTE: this is subtle as a TB may span two physical pages */
983 /* NOTE: tb_end may be after the end of the page, but
984 it is not a problem */
985 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
986 tb_end
= tb_start
+ tb
->size
;
987 if (tb_end
> TARGET_PAGE_SIZE
) {
988 tb_end
= TARGET_PAGE_SIZE
;
992 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
994 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
995 tb
= tb
->page_next
[n
];
999 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1000 target_ulong pc
, target_ulong cs_base
,
1001 int flags
, int cflags
)
1003 CPUArchState
*env
= cpu
->env_ptr
;
1004 TranslationBlock
*tb
;
1005 tb_page_addr_t phys_pc
, phys_page2
;
1006 target_ulong virt_page2
;
1009 phys_pc
= get_page_addr_code(env
, pc
);
1011 cflags
|= CF_USE_ICOUNT
;
1015 /* flush must be done */
1017 /* cannot fail at this point */
1019 /* Don't forget to invalidate previous TB info. */
1020 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
1022 tb
->tc_ptr
= tcg_ctx
.code_gen_ptr
;
1023 tb
->cs_base
= cs_base
;
1025 tb
->cflags
= cflags
;
1026 cpu_gen_code(env
, tb
, &code_gen_size
);
1027 tcg_ctx
.code_gen_ptr
= (void *)(((uintptr_t)tcg_ctx
.code_gen_ptr
+
1028 code_gen_size
+ CODE_GEN_ALIGN
- 1) & ~(CODE_GEN_ALIGN
- 1));
1030 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1031 /* if we are doing vsyscall don't link the page as it lies in high memory
1032 and tb_alloc_page will abort due to page_l1_map returning NULL */
1033 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1034 && phys_pc
< TARGET_VSYSCALL_END
))
1038 /* check next page if needed */
1039 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1041 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1042 phys_page2
= get_page_addr_code(env
, virt_page2
);
1044 tb_link_page(tb
, phys_pc
, phys_page2
);
1049 * Invalidate all TBs which intersect with the target physical address range
1050 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1051 * 'is_cpu_write_access' should be true if called from a real cpu write
1052 * access: the virtual CPU will exit the current TB if code is modified inside
1055 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1057 while (start
< end
) {
1058 tb_invalidate_phys_page_range(start
, end
, 0);
1059 start
&= TARGET_PAGE_MASK
;
1060 start
+= TARGET_PAGE_SIZE
;
1065 * Invalidate all TBs which intersect with the target physical address range
1066 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1067 * 'is_cpu_write_access' should be true if called from a real cpu write
1068 * access: the virtual CPU will exit the current TB if code is modified inside
1071 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1072 int is_cpu_write_access
)
1074 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1075 CPUState
*cpu
= current_cpu
;
1076 #if defined(TARGET_HAS_PRECISE_SMC)
1077 CPUArchState
*env
= NULL
;
1079 tb_page_addr_t tb_start
, tb_end
;
1082 #ifdef TARGET_HAS_PRECISE_SMC
1083 int current_tb_not_found
= is_cpu_write_access
;
1084 TranslationBlock
*current_tb
= NULL
;
1085 int current_tb_modified
= 0;
1086 target_ulong current_pc
= 0;
1087 target_ulong current_cs_base
= 0;
1088 int current_flags
= 0;
1089 #endif /* TARGET_HAS_PRECISE_SMC */
1091 p
= page_find(start
>> TARGET_PAGE_BITS
);
1095 #if defined(TARGET_HAS_PRECISE_SMC)
1101 /* we remove all the TBs in the range [start, end[ */
1102 /* XXX: see if in some cases it could be faster to invalidate all
1105 while (tb
!= NULL
) {
1106 n
= (uintptr_t)tb
& 3;
1107 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1108 tb_next
= tb
->page_next
[n
];
1109 /* NOTE: this is subtle as a TB may span two physical pages */
1111 /* NOTE: tb_end may be after the end of the page, but
1112 it is not a problem */
1113 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1114 tb_end
= tb_start
+ tb
->size
;
1116 tb_start
= tb
->page_addr
[1];
1117 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1119 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1120 #ifdef TARGET_HAS_PRECISE_SMC
1121 if (current_tb_not_found
) {
1122 current_tb_not_found
= 0;
1124 if (cpu
->mem_io_pc
) {
1125 /* now we have a real cpu fault */
1126 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1129 if (current_tb
== tb
&&
1130 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1131 /* If we are modifying the current TB, we must stop
1132 its execution. We could be more precise by checking
1133 that the modification is after the current PC, but it
1134 would require a specialized function to partially
1135 restore the CPU state */
1137 current_tb_modified
= 1;
1138 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1139 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1142 #endif /* TARGET_HAS_PRECISE_SMC */
1143 /* we need to do that to handle the case where a signal
1144 occurs while doing tb_phys_invalidate() */
1147 saved_tb
= cpu
->current_tb
;
1148 cpu
->current_tb
= NULL
;
1150 tb_phys_invalidate(tb
, -1);
1152 cpu
->current_tb
= saved_tb
;
1153 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1154 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1160 #if !defined(CONFIG_USER_ONLY)
1161 /* if no code remaining, no need to continue to use slow writes */
1163 invalidate_page_bitmap(p
);
1164 tlb_unprotect_code(start
);
1167 #ifdef TARGET_HAS_PRECISE_SMC
1168 if (current_tb_modified
) {
1169 /* we generate a block containing just the instruction
1170 modifying the memory. It will ensure that it cannot modify
1172 cpu
->current_tb
= NULL
;
1173 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1174 cpu_resume_from_signal(cpu
, NULL
);
1179 /* len must be <= 8 and start must be a multiple of len */
1180 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1186 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1187 cpu_single_env
->mem_io_vaddr
, len
,
1188 cpu_single_env
->eip
,
1189 cpu_single_env
->eip
+
1190 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1193 p
= page_find(start
>> TARGET_PAGE_BITS
);
1197 if (!p
->code_bitmap
&&
1198 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1199 /* build code bitmap */
1200 build_page_bitmap(p
);
1202 if (p
->code_bitmap
) {
1206 nr
= start
& ~TARGET_PAGE_MASK
;
1207 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1208 if (b
& ((1 << len
) - 1)) {
1213 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1217 #if !defined(CONFIG_SOFTMMU)
1218 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1219 uintptr_t pc
, void *puc
,
1222 TranslationBlock
*tb
;
1225 #ifdef TARGET_HAS_PRECISE_SMC
1226 TranslationBlock
*current_tb
= NULL
;
1227 CPUState
*cpu
= current_cpu
;
1228 CPUArchState
*env
= NULL
;
1229 int current_tb_modified
= 0;
1230 target_ulong current_pc
= 0;
1231 target_ulong current_cs_base
= 0;
1232 int current_flags
= 0;
1235 addr
&= TARGET_PAGE_MASK
;
1236 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1241 #ifdef TARGET_HAS_PRECISE_SMC
1242 if (tb
&& pc
!= 0) {
1243 current_tb
= tb_find_pc(pc
);
1249 while (tb
!= NULL
) {
1250 n
= (uintptr_t)tb
& 3;
1251 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1252 #ifdef TARGET_HAS_PRECISE_SMC
1253 if (current_tb
== tb
&&
1254 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1255 /* If we are modifying the current TB, we must stop
1256 its execution. We could be more precise by checking
1257 that the modification is after the current PC, but it
1258 would require a specialized function to partially
1259 restore the CPU state */
1261 current_tb_modified
= 1;
1262 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1263 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1266 #endif /* TARGET_HAS_PRECISE_SMC */
1267 tb_phys_invalidate(tb
, addr
);
1268 tb
= tb
->page_next
[n
];
1271 #ifdef TARGET_HAS_PRECISE_SMC
1272 if (current_tb_modified
) {
1273 /* we generate a block containing just the instruction
1274 modifying the memory. It will ensure that it cannot modify
1276 cpu
->current_tb
= NULL
;
1277 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1281 cpu_resume_from_signal(cpu
, puc
);
1287 /* add the tb in the target page and protect it if necessary */
1288 static inline void tb_alloc_page(TranslationBlock
*tb
,
1289 unsigned int n
, tb_page_addr_t page_addr
)
1292 #ifndef CONFIG_USER_ONLY
1293 bool page_already_protected
;
1296 tb
->page_addr
[n
] = page_addr
;
1297 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1298 tb
->page_next
[n
] = p
->first_tb
;
1299 #ifndef CONFIG_USER_ONLY
1300 page_already_protected
= p
->first_tb
!= NULL
;
1302 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1303 invalidate_page_bitmap(p
);
1305 #if defined(CONFIG_USER_ONLY)
1306 if (p
->flags
& PAGE_WRITE
) {
1311 /* force the host page as non writable (writes will have a
1312 page fault + mprotect overhead) */
1313 page_addr
&= qemu_host_page_mask
;
1315 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1316 addr
+= TARGET_PAGE_SIZE
) {
1318 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1323 p2
->flags
&= ~PAGE_WRITE
;
1325 mprotect(g2h(page_addr
), qemu_host_page_size
,
1326 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1327 #ifdef DEBUG_TB_INVALIDATE
1328 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1333 /* if some code is already present, then the pages are already
1334 protected. So we handle the case where only the first TB is
1335 allocated in a physical page */
1336 if (!page_already_protected
) {
1337 tlb_protect_code(page_addr
);
1342 /* add a new TB and link it to the physical page tables. phys_page2 is
1343 (-1) to indicate that only one page contains the TB. */
1344 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1345 tb_page_addr_t phys_page2
)
1348 TranslationBlock
**ptb
;
1350 /* Grab the mmap lock to stop another thread invalidating this TB
1351 before we are done. */
1353 /* add in the physical hash table */
1354 h
= tb_phys_hash_func(phys_pc
);
1355 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1356 tb
->phys_hash_next
= *ptb
;
1359 /* add in the page list */
1360 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1361 if (phys_page2
!= -1) {
1362 tb_alloc_page(tb
, 1, phys_page2
);
1364 tb
->page_addr
[1] = -1;
1367 tb
->jmp_first
= (TranslationBlock
*)((uintptr_t)tb
| 2);
1368 tb
->jmp_next
[0] = NULL
;
1369 tb
->jmp_next
[1] = NULL
;
1371 /* init original jump addresses */
1372 if (tb
->tb_next_offset
[0] != 0xffff) {
1373 tb_reset_jump(tb
, 0);
1375 if (tb
->tb_next_offset
[1] != 0xffff) {
1376 tb_reset_jump(tb
, 1);
1379 #ifdef DEBUG_TB_CHECK
1385 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1386 tb[1].tc_ptr. Return NULL if not found */
1387 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1389 int m_min
, m_max
, m
;
1391 TranslationBlock
*tb
;
1393 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1396 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1397 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1400 /* binary search (cf Knuth) */
1402 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1403 while (m_min
<= m_max
) {
1404 m
= (m_min
+ m_max
) >> 1;
1405 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1406 v
= (uintptr_t)tb
->tc_ptr
;
1409 } else if (tc_ptr
< v
) {
1415 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1418 #if !defined(CONFIG_USER_ONLY)
1419 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1421 ram_addr_t ram_addr
;
1426 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1427 if (!(memory_region_is_ram(mr
)
1428 || memory_region_is_romd(mr
))) {
1432 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1434 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1437 #endif /* !defined(CONFIG_USER_ONLY) */
1439 void tb_check_watchpoint(CPUState
*cpu
)
1441 TranslationBlock
*tb
;
1443 tb
= tb_find_pc(cpu
->mem_io_pc
);
1445 /* We can use retranslation to find the PC. */
1446 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1447 tb_phys_invalidate(tb
, -1);
1449 /* The exception probably happened in a helper. The CPU state should
1450 have been saved before calling it. Fetch the PC from there. */
1451 CPUArchState
*env
= cpu
->env_ptr
;
1452 target_ulong pc
, cs_base
;
1453 tb_page_addr_t addr
;
1456 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1457 addr
= get_page_addr_code(env
, pc
);
1458 tb_invalidate_phys_range(addr
, addr
+ 1);
1462 #ifndef CONFIG_USER_ONLY
1463 /* mask must never be zero, except for A20 change call */
1464 static void tcg_handle_interrupt(CPUState
*cpu
, int mask
)
1468 old_mask
= cpu
->interrupt_request
;
1469 cpu
->interrupt_request
|= mask
;
1472 * If called from iothread context, wake the target cpu in
1475 if (!qemu_cpu_is_self(cpu
)) {
1481 cpu
->icount_decr
.u16
.high
= 0xffff;
1483 && (mask
& ~old_mask
) != 0) {
1484 cpu_abort(cpu
, "Raised interrupt while not in I/O function");
1487 cpu
->tcg_exit_req
= 1;
1491 CPUInterruptHandler cpu_interrupt_handler
= tcg_handle_interrupt
;
1493 /* in deterministic execution mode, instructions doing device I/Os
1494 must be at the end of the TB */
1495 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1497 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1498 CPUArchState
*env
= cpu
->env_ptr
;
1500 TranslationBlock
*tb
;
1502 target_ulong pc
, cs_base
;
1505 tb
= tb_find_pc(retaddr
);
1507 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1510 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1511 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1512 /* Calculate how many instructions had been executed before the fault
1514 n
= n
- cpu
->icount_decr
.u16
.low
;
1515 /* Generate a new TB ending on the I/O insn. */
1517 /* On MIPS and SH, delay slot instructions can only be restarted if
1518 they were already the first instruction in the TB. If this is not
1519 the first instruction in a TB then re-execute the preceding
1521 #if defined(TARGET_MIPS)
1522 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1523 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1524 cpu
->icount_decr
.u16
.low
++;
1525 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1527 #elif defined(TARGET_SH4)
1528 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1531 cpu
->icount_decr
.u16
.low
++;
1532 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1535 /* This should never happen. */
1536 if (n
> CF_COUNT_MASK
) {
1537 cpu_abort(cpu
, "TB too big during recompile");
1540 cflags
= n
| CF_LAST_IO
;
1542 cs_base
= tb
->cs_base
;
1544 tb_phys_invalidate(tb
, -1);
1545 if (tb
->cflags
& CF_NOCACHE
) {
1547 /* Invalidate original TB if this TB was generated in
1548 * cpu_exec_nocache() */
1549 tb_phys_invalidate(tb
->orig_tb
, -1);
1553 /* FIXME: In theory this could raise an exception. In practice
1554 we have already translated the block once so it's probably ok. */
1555 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1556 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1557 the first in the TB) then we end up generating a whole new TB and
1558 repeating the fault, which is horribly inefficient.
1559 Better would be to execute just this insn uncached, or generate a
1561 cpu_resume_from_signal(cpu
, NULL
);
1564 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1568 /* Discard jump cache entries for any tb which might potentially
1569 overlap the flushed page. */
1570 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1571 memset(&cpu
->tb_jmp_cache
[i
], 0,
1572 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1574 i
= tb_jmp_cache_hash_page(addr
);
1575 memset(&cpu
->tb_jmp_cache
[i
], 0,
1576 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1579 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1581 int i
, target_code_size
, max_target_code_size
;
1582 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1583 TranslationBlock
*tb
;
1585 target_code_size
= 0;
1586 max_target_code_size
= 0;
1588 direct_jmp_count
= 0;
1589 direct_jmp2_count
= 0;
1590 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1591 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1592 target_code_size
+= tb
->size
;
1593 if (tb
->size
> max_target_code_size
) {
1594 max_target_code_size
= tb
->size
;
1596 if (tb
->page_addr
[1] != -1) {
1599 if (tb
->tb_next_offset
[0] != 0xffff) {
1601 if (tb
->tb_next_offset
[1] != 0xffff) {
1602 direct_jmp2_count
++;
1606 /* XXX: avoid using doubles ? */
1607 cpu_fprintf(f
, "Translation buffer state:\n");
1608 cpu_fprintf(f
, "gen code size %td/%zd\n",
1609 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1610 tcg_ctx
.code_gen_buffer_max_size
);
1611 cpu_fprintf(f
, "TB count %d/%d\n",
1612 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1613 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1614 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1615 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1616 max_target_code_size
);
1617 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1618 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1619 tcg_ctx
.code_gen_buffer
) /
1620 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1621 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1622 tcg_ctx
.code_gen_buffer
) /
1623 target_code_size
: 0);
1624 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1625 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1626 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1627 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1629 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1630 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1632 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1633 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1634 cpu_fprintf(f
, "\nStatistics:\n");
1635 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1636 cpu_fprintf(f
, "TB invalidate count %d\n",
1637 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1638 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1639 tcg_dump_info(f
, cpu_fprintf
);
1642 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1644 tcg_dump_op_count(f
, cpu_fprintf
);
1647 #else /* CONFIG_USER_ONLY */
1649 void cpu_interrupt(CPUState
*cpu
, int mask
)
1651 cpu
->interrupt_request
|= mask
;
1652 cpu
->tcg_exit_req
= 1;
1656 * Walks guest process memory "regions" one by one
1657 * and calls callback function 'fn' for each region.
1659 struct walk_memory_regions_data
{
1660 walk_memory_regions_fn fn
;
1666 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1667 target_ulong end
, int new_prot
)
1669 if (data
->start
!= -1u) {
1670 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1676 data
->start
= (new_prot
? end
: -1u);
1677 data
->prot
= new_prot
;
1682 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1683 target_ulong base
, int level
, void **lp
)
1689 return walk_memory_regions_end(data
, base
, 0);
1695 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1696 int prot
= pd
[i
].flags
;
1698 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1699 if (prot
!= data
->prot
) {
1700 rc
= walk_memory_regions_end(data
, pa
, prot
);
1709 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1710 pa
= base
| ((target_ulong
)i
<<
1711 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1712 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1722 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1724 struct walk_memory_regions_data data
;
1732 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1733 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1734 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1740 return walk_memory_regions_end(&data
, 0, 0);
1743 static int dump_region(void *priv
, target_ulong start
,
1744 target_ulong end
, abi_ulong prot
)
1746 FILE *f
= (FILE *)priv
;
1748 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1749 " "TARGET_FMT_lx
" %c%c%c\n",
1750 start
, end
, end
- start
,
1751 ((prot
& PAGE_READ
) ? 'r' : '-'),
1752 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1753 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1758 /* dump memory mappings */
1759 void page_dump(FILE *f
)
1761 const int length
= sizeof(target_ulong
) * 2;
1762 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1763 length
, "start", length
, "end", length
, "size", "prot");
1764 walk_memory_regions(f
, dump_region
);
1767 int page_get_flags(target_ulong address
)
1771 p
= page_find(address
>> TARGET_PAGE_BITS
);
1778 /* Modify the flags of a page and invalidate the code if necessary.
1779 The flag PAGE_WRITE_ORG is positioned automatically depending
1780 on PAGE_WRITE. The mmap_lock should already be held. */
1781 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1783 target_ulong addr
, len
;
1785 /* This function should never be called with addresses outside the
1786 guest address space. If this assert fires, it probably indicates
1787 a missing call to h2g_valid. */
1788 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1789 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1791 assert(start
< end
);
1793 start
= start
& TARGET_PAGE_MASK
;
1794 end
= TARGET_PAGE_ALIGN(end
);
1796 if (flags
& PAGE_WRITE
) {
1797 flags
|= PAGE_WRITE_ORG
;
1800 for (addr
= start
, len
= end
- start
;
1802 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1803 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1805 /* If the write protection bit is set, then we invalidate
1807 if (!(p
->flags
& PAGE_WRITE
) &&
1808 (flags
& PAGE_WRITE
) &&
1810 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1816 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1822 /* This function should never be called with addresses outside the
1823 guest address space. If this assert fires, it probably indicates
1824 a missing call to h2g_valid. */
1825 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1826 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1832 if (start
+ len
- 1 < start
) {
1833 /* We've wrapped around. */
1837 /* must do before we loose bits in the next step */
1838 end
= TARGET_PAGE_ALIGN(start
+ len
);
1839 start
= start
& TARGET_PAGE_MASK
;
1841 for (addr
= start
, len
= end
- start
;
1843 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1844 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1848 if (!(p
->flags
& PAGE_VALID
)) {
1852 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1855 if (flags
& PAGE_WRITE
) {
1856 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1859 /* unprotect the page if it was put read-only because it
1860 contains translated code */
1861 if (!(p
->flags
& PAGE_WRITE
)) {
1862 if (!page_unprotect(addr
, 0, NULL
)) {
1871 /* called from signal handler: invalidate the code and unprotect the
1872 page. Return TRUE if the fault was successfully handled. */
1873 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1877 target_ulong host_start
, host_end
, addr
;
1879 /* Technically this isn't safe inside a signal handler. However we
1880 know this only ever happens in a synchronous SEGV handler, so in
1881 practice it seems to be ok. */
1884 p
= page_find(address
>> TARGET_PAGE_BITS
);
1890 /* if the page was really writable, then we change its
1891 protection back to writable */
1892 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
1893 host_start
= address
& qemu_host_page_mask
;
1894 host_end
= host_start
+ qemu_host_page_size
;
1897 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
1898 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1899 p
->flags
|= PAGE_WRITE
;
1902 /* and since the content will be modified, we must invalidate
1903 the corresponding translated code. */
1904 tb_invalidate_phys_page(addr
, pc
, puc
, true);
1905 #ifdef DEBUG_TB_CHECK
1906 tb_invalidate_check(addr
);
1909 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
1918 #endif /* CONFIG_USER_ONLY */