4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "qemu/osdep.h"
24 #include "qemu-common.h"
25 #define NO_CPU_IO_DEFS
28 #include "disas/disas.h"
29 #include "exec/exec-all.h"
31 #if defined(CONFIG_USER_ONLY)
33 #if defined(TARGET_X86_64)
36 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
37 #include <sys/param.h>
38 #if __FreeBSD_version >= 700104
39 #define HAVE_KINFO_GETVMMAP
40 #define sigqueue sigqueue_freebsd /* avoid redefinition */
42 #include <machine/profile.h>
51 #include "exec/address-spaces.h"
54 #include "exec/cputlb.h"
55 #include "exec/tb-hash.h"
56 #include "translate-all.h"
57 #include "qemu/bitmap.h"
58 #include "qemu/timer.h"
61 //#define DEBUG_TB_INVALIDATE
63 /* make various TB consistency checks */
64 //#define DEBUG_TB_CHECK
66 #if !defined(CONFIG_USER_ONLY)
67 /* TB consistency checks only implemented for usermode emulation. */
71 #define SMC_BITMAP_USE_THRESHOLD 10
73 typedef struct PageDesc
{
74 /* list of TBs intersecting this ram page */
75 TranslationBlock
*first_tb
;
77 /* in order to optimize self modifying code, we count the number
78 of lookups we do to a given page to use a bitmap */
79 unsigned int code_write_count
;
80 unsigned long *code_bitmap
;
86 /* In system mode we want L1_MAP to be based on ram offsets,
87 while in user mode we want it to be based on virtual addresses. */
88 #if !defined(CONFIG_USER_ONLY)
89 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
90 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
95 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
98 /* Size of the L2 (and L3, etc) page tables. */
100 #define V_L2_SIZE (1 << V_L2_BITS)
102 /* The bits remaining after N lower levels of page tables. */
103 #define V_L1_BITS_REM \
104 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
106 #if V_L1_BITS_REM < 4
107 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
109 #define V_L1_BITS V_L1_BITS_REM
112 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
114 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
116 uintptr_t qemu_host_page_size
;
117 intptr_t qemu_host_page_mask
;
119 /* The bottom level has pointers to PageDesc */
120 static void *l1_map
[V_L1_SIZE
];
122 /* code generation context */
125 /* translation block context */
126 #ifdef CONFIG_USER_ONLY
127 __thread
int have_tb_lock
;
132 #ifdef CONFIG_USER_ONLY
133 assert(!have_tb_lock
);
134 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
141 #ifdef CONFIG_USER_ONLY
142 assert(have_tb_lock
);
144 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
148 void tb_lock_reset(void)
150 #ifdef CONFIG_USER_ONLY
152 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
158 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
160 void cpu_gen_init(void)
162 tcg_context_init(&tcg_ctx
);
165 /* Encode VAL as a signed leb128 sequence at P.
166 Return P incremented past the encoded value. */
167 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
174 more
= !((val
== 0 && (byte
& 0x40) == 0)
175 || (val
== -1 && (byte
& 0x40) != 0));
185 /* Decode a signed leb128 sequence at *PP; increment *PP past the
186 decoded value. Return the decoded value. */
187 static target_long
decode_sleb128(uint8_t **pp
)
195 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
197 } while (byte
& 0x80);
198 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
199 val
|= -(target_ulong
)1 << shift
;
206 /* Encode the data collected about the instructions while compiling TB.
207 Place the data at BLOCK, and return the number of bytes consumed.
209 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
210 which come from the target's insn_start data, followed by a uintptr_t
211 which comes from the host pc of the end of the code implementing the insn.
213 Each line of the table is encoded as sleb128 deltas from the previous
214 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
215 That is, the first column is seeded with the guest pc, the last column
216 with the host pc, and the middle columns with zeros. */
218 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
220 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
224 tb
->tc_search
= block
;
226 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
229 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
231 prev
= (j
== 0 ? tb
->pc
: 0);
233 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
235 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
237 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
238 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
240 /* Test for (pending) buffer overflow. The assumption is that any
241 one row beginning below the high water mark cannot overrun
242 the buffer completely. Thus we can test for overflow after
243 encoding a row without having to check during encoding. */
244 if (unlikely(p
> highwater
)) {
252 /* The cpu state corresponding to 'searched_pc' is restored. */
253 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
254 uintptr_t searched_pc
)
256 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
257 uintptr_t host_pc
= (uintptr_t)tb
->tc_ptr
;
258 CPUArchState
*env
= cpu
->env_ptr
;
259 uint8_t *p
= tb
->tc_search
;
260 int i
, j
, num_insns
= tb
->icount
;
261 #ifdef CONFIG_PROFILER
262 int64_t ti
= profile_getclock();
265 searched_pc
-= GETPC_ADJ
;
267 if (searched_pc
< host_pc
) {
271 /* Reconstruct the stored insn data while looking for the point at
272 which the end of the insn exceeds the searched_pc. */
273 for (i
= 0; i
< num_insns
; ++i
) {
274 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
275 data
[j
] += decode_sleb128(&p
);
277 host_pc
+= decode_sleb128(&p
);
278 if (host_pc
> searched_pc
) {
285 if (tb
->cflags
& CF_USE_ICOUNT
) {
287 /* Reset the cycle counter to the start of the block. */
288 cpu
->icount_decr
.u16
.low
+= num_insns
;
289 /* Clear the IO flag. */
292 cpu
->icount_decr
.u16
.low
-= i
;
293 restore_state_to_opc(env
, tb
, data
);
295 #ifdef CONFIG_PROFILER
296 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
297 tcg_ctx
.restore_count
++;
302 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
304 TranslationBlock
*tb
;
306 tb
= tb_find_pc(retaddr
);
308 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
309 if (tb
->cflags
& CF_NOCACHE
) {
310 /* one-shot translation, invalidate it immediately */
311 tb_phys_invalidate(tb
, -1);
319 void page_size_init(void)
321 /* NOTE: we can always suppose that qemu_host_page_size >=
323 qemu_real_host_page_size
= getpagesize();
324 qemu_real_host_page_mask
= -(intptr_t)qemu_real_host_page_size
;
325 if (qemu_host_page_size
== 0) {
326 qemu_host_page_size
= qemu_real_host_page_size
;
328 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
329 qemu_host_page_size
= TARGET_PAGE_SIZE
;
331 qemu_host_page_mask
= -(intptr_t)qemu_host_page_size
;
334 static void page_init(void)
337 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
339 #ifdef HAVE_KINFO_GETVMMAP
340 struct kinfo_vmentry
*freep
;
343 freep
= kinfo_getvmmap(getpid(), &cnt
);
346 for (i
= 0; i
< cnt
; i
++) {
347 unsigned long startaddr
, endaddr
;
349 startaddr
= freep
[i
].kve_start
;
350 endaddr
= freep
[i
].kve_end
;
351 if (h2g_valid(startaddr
)) {
352 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
354 if (h2g_valid(endaddr
)) {
355 endaddr
= h2g(endaddr
);
356 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
358 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
360 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
371 last_brk
= (unsigned long)sbrk(0);
373 f
= fopen("/compat/linux/proc/self/maps", "r");
378 unsigned long startaddr
, endaddr
;
381 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
383 if (n
== 2 && h2g_valid(startaddr
)) {
384 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
386 if (h2g_valid(endaddr
)) {
387 endaddr
= h2g(endaddr
);
391 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
404 * Called with mmap_lock held for user-mode emulation.
406 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
412 /* Level 1. Always allocated. */
413 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
416 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
417 void **p
= atomic_rcu_read(lp
);
423 p
= g_new0(void *, V_L2_SIZE
);
424 atomic_rcu_set(lp
, p
);
427 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
430 pd
= atomic_rcu_read(lp
);
435 pd
= g_new0(PageDesc
, V_L2_SIZE
);
436 atomic_rcu_set(lp
, pd
);
439 return pd
+ (index
& (V_L2_SIZE
- 1));
442 static inline PageDesc
*page_find(tb_page_addr_t index
)
444 return page_find_alloc(index
, 0);
447 #if defined(CONFIG_USER_ONLY)
448 /* Currently it is not recommended to allocate big chunks of data in
449 user mode. It will change when a dedicated libc will be used. */
450 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
451 region in which the guest needs to run. Revisit this. */
452 #define USE_STATIC_CODE_GEN_BUFFER
455 /* Minimum size of the code gen buffer. This number is randomly chosen,
456 but not so small that we can't have a fair number of TB's live. */
457 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
459 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
460 indicated, this is constrained by the range of direct branches on the
461 host cpu, as used by the TCG implementation of goto_tb. */
462 #if defined(__x86_64__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__sparc__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
466 #elif defined(__powerpc64__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
468 #elif defined(__powerpc__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
470 #elif defined(__aarch64__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
472 #elif defined(__arm__)
473 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
474 #elif defined(__s390x__)
475 /* We have a +- 4GB range on the branches; leave some slop. */
476 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
477 #elif defined(__mips__)
478 /* We have a 256MB branch region, but leave room to make sure the
479 main executable is also within that region. */
480 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
482 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
485 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
487 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
488 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
489 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
491 static inline size_t size_code_gen_buffer(size_t tb_size
)
493 /* Size the buffer. */
495 #ifdef USE_STATIC_CODE_GEN_BUFFER
496 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
498 /* ??? Needs adjustments. */
499 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
500 static buffer, we could size this on RESERVED_VA, on the text
501 segment size of the executable, or continue to use the default. */
502 tb_size
= (unsigned long)(ram_size
/ 4);
505 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
506 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
508 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
509 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
515 /* In order to use J and JAL within the code_gen_buffer, we require
516 that the buffer not cross a 256MB boundary. */
517 static inline bool cross_256mb(void *addr
, size_t size
)
519 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
522 /* We weren't able to allocate a buffer without crossing that boundary,
523 so make do with the larger portion of the buffer that doesn't cross.
524 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
525 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
527 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
528 size_t size2
= buf1
+ size1
- buf2
;
536 tcg_ctx
.code_gen_buffer_size
= size1
;
541 #ifdef USE_STATIC_CODE_GEN_BUFFER
542 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
543 __attribute__((aligned(CODE_GEN_ALIGN
)));
546 static inline void do_protect(void *addr
, long size
, int prot
)
549 VirtualProtect(addr
, size
, prot
, &old_protect
);
552 static inline void map_exec(void *addr
, long size
)
554 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
557 static inline void map_none(void *addr
, long size
)
559 do_protect(addr
, size
, PAGE_NOACCESS
);
562 static inline void do_protect(void *addr
, long size
, int prot
)
564 uintptr_t start
, end
;
566 start
= (uintptr_t)addr
;
567 start
&= qemu_real_host_page_mask
;
569 end
= (uintptr_t)addr
+ size
;
570 end
= ROUND_UP(end
, qemu_real_host_page_size
);
572 mprotect((void *)start
, end
- start
, prot
);
575 static inline void map_exec(void *addr
, long size
)
577 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
580 static inline void map_none(void *addr
, long size
)
582 do_protect(addr
, size
, PROT_NONE
);
586 static inline void *alloc_code_gen_buffer(void)
588 void *buf
= static_code_gen_buffer
;
589 size_t full_size
, size
;
591 /* The size of the buffer, rounded down to end on a page boundary. */
592 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
593 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
595 /* Reserve a guard page. */
596 size
= full_size
- qemu_real_host_page_size
;
598 /* Honor a command-line option limiting the size of the buffer. */
599 if (size
> tcg_ctx
.code_gen_buffer_size
) {
600 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
601 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
603 tcg_ctx
.code_gen_buffer_size
= size
;
606 if (cross_256mb(buf
, size
)) {
607 buf
= split_cross_256mb(buf
, size
);
608 size
= tcg_ctx
.code_gen_buffer_size
;
613 map_none(buf
+ size
, qemu_real_host_page_size
);
614 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
618 #elif defined(_WIN32)
619 static inline void *alloc_code_gen_buffer(void)
621 size_t size
= tcg_ctx
.code_gen_buffer_size
;
624 /* Perform the allocation in two steps, so that the guard page
625 is reserved but uncommitted. */
626 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
627 MEM_RESERVE
, PAGE_NOACCESS
);
629 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
630 assert(buf1
== buf2
);
636 static inline void *alloc_code_gen_buffer(void)
638 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
640 size_t size
= tcg_ctx
.code_gen_buffer_size
;
643 /* Constrain the position of the buffer based on the host cpu.
644 Note that these addresses are chosen in concert with the
645 addresses assigned in the relevant linker script file. */
646 # if defined(__PIE__) || defined(__PIC__)
647 /* Don't bother setting a preferred location if we're building
648 a position-independent executable. We're more likely to get
649 an address near the main executable if we let the kernel
650 choose the address. */
651 # elif defined(__x86_64__) && defined(MAP_32BIT)
652 /* Force the memory down into low memory with the executable.
653 Leave the choice of exact location with the kernel. */
655 /* Cannot expect to map more than 800MB in low memory. */
656 if (size
> 800u * 1024 * 1024) {
657 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
659 # elif defined(__sparc__)
660 start
= 0x40000000ul
;
661 # elif defined(__s390x__)
662 start
= 0x90000000ul
;
663 # elif defined(__mips__)
664 # if _MIPS_SIM == _ABI64
665 start
= 0x128000000ul
;
667 start
= 0x08000000ul
;
671 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
672 PROT_NONE
, flags
, -1, 0);
673 if (buf
== MAP_FAILED
) {
678 if (cross_256mb(buf
, size
)) {
679 /* Try again, with the original still mapped, to avoid re-acquiring
680 that 256mb crossing. This time don't specify an address. */
682 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
683 PROT_NONE
, flags
, -1, 0);
684 switch (buf2
!= MAP_FAILED
) {
686 if (!cross_256mb(buf2
, size
)) {
687 /* Success! Use the new buffer. */
688 munmap(buf
, size
+ qemu_real_host_page_size
);
691 /* Failure. Work with what we had. */
692 munmap(buf2
, size
+ qemu_real_host_page_size
);
695 /* Split the original buffer. Free the smaller half. */
696 buf2
= split_cross_256mb(buf
, size
);
697 size2
= tcg_ctx
.code_gen_buffer_size
;
699 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
701 munmap(buf
, size
- size2
);
710 /* Make the final buffer accessible. The guard page at the end
711 will remain inaccessible with PROT_NONE. */
712 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
714 /* Request large pages for the buffer. */
715 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
719 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
721 static inline void code_gen_alloc(size_t tb_size
)
723 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
724 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
725 if (tcg_ctx
.code_gen_buffer
== NULL
) {
726 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
730 /* Estimate a good size for the number of TBs we can support. We
731 still haven't deducted the prologue from the buffer size here,
732 but that's minimal and won't affect the estimate much. */
733 tcg_ctx
.code_gen_max_blocks
734 = tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
735 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
, tcg_ctx
.code_gen_max_blocks
);
737 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
740 static void tb_htable_init(void)
742 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
744 qht_init(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
, mode
);
747 /* Must be called before using the QEMU cpus. 'tb_size' is the size
748 (in bytes) allocated to the translation buffer. Zero means default
750 void tcg_exec_init(uintptr_t tb_size
)
755 code_gen_alloc(tb_size
);
756 #if defined(CONFIG_SOFTMMU)
757 /* There's no guest base to take into account, so go ahead and
758 initialize the prologue now. */
759 tcg_prologue_init(&tcg_ctx
);
763 bool tcg_enabled(void)
765 return tcg_ctx
.code_gen_buffer
!= NULL
;
768 /* Allocate a new translation block. Flush the translation buffer if
769 too many translation blocks or too much generated code. */
770 static TranslationBlock
*tb_alloc(target_ulong pc
)
772 TranslationBlock
*tb
;
774 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
) {
777 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
784 void tb_free(TranslationBlock
*tb
)
786 /* In practice this is mostly used for single use temporary TB
787 Ignore the hard cases and just back up if this TB happens to
788 be the last one generated. */
789 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
790 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
791 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
792 tcg_ctx
.tb_ctx
.nb_tbs
--;
796 static inline void invalidate_page_bitmap(PageDesc
*p
)
798 #ifdef CONFIG_SOFTMMU
799 g_free(p
->code_bitmap
);
800 p
->code_bitmap
= NULL
;
801 p
->code_write_count
= 0;
805 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
806 static void page_flush_tb_1(int level
, void **lp
)
816 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
817 pd
[i
].first_tb
= NULL
;
818 invalidate_page_bitmap(pd
+ i
);
823 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
824 page_flush_tb_1(level
- 1, pp
+ i
);
829 static void page_flush_tb(void)
833 for (i
= 0; i
< V_L1_SIZE
; i
++) {
834 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
838 /* flush all the translation blocks */
839 static void do_tb_flush(CPUState
*cpu
, void *data
)
841 unsigned tb_flush_req
= (unsigned) (uintptr_t) data
;
845 /* If it's already been done on request of another CPU,
848 if (tcg_ctx
.tb_ctx
.tb_flush_count
!= tb_flush_req
) {
852 #if defined(DEBUG_FLUSH)
853 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
854 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
855 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
856 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
857 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
859 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
860 > tcg_ctx
.code_gen_buffer_size
) {
861 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
867 for (i
= 0; i
< TB_JMP_CACHE_SIZE
; ++i
) {
868 atomic_set(&cpu
->tb_jmp_cache
[i
], NULL
);
872 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
873 qht_reset_size(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
876 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
877 /* XXX: flush processor icache at this point if cache flush is
879 atomic_mb_set(&tcg_ctx
.tb_ctx
.tb_flush_count
,
880 tcg_ctx
.tb_ctx
.tb_flush_count
+ 1);
886 void tb_flush(CPUState
*cpu
)
889 uintptr_t tb_flush_req
= atomic_mb_read(&tcg_ctx
.tb_ctx
.tb_flush_count
);
890 async_safe_run_on_cpu(cpu
, do_tb_flush
, (void *) tb_flush_req
);
894 #ifdef DEBUG_TB_CHECK
897 do_tb_invalidate_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
899 TranslationBlock
*tb
= p
;
900 target_ulong addr
= *(target_ulong
*)userp
;
902 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
903 printf("ERROR invalidate: address=" TARGET_FMT_lx
904 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
908 static void tb_invalidate_check(target_ulong address
)
910 address
&= TARGET_PAGE_MASK
;
911 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
915 do_tb_page_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
917 TranslationBlock
*tb
= p
;
920 flags1
= page_get_flags(tb
->pc
);
921 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
922 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
923 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
924 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
928 /* verify that all the pages have correct rights for code */
929 static void tb_page_check(void)
931 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_page_check
, NULL
);
936 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
938 TranslationBlock
*tb1
;
943 n1
= (uintptr_t)tb1
& 3;
944 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
946 *ptb
= tb1
->page_next
[n1
];
949 ptb
= &tb1
->page_next
[n1
];
953 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
954 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
956 TranslationBlock
*tb1
;
960 ptb
= &tb
->jmp_list_next
[n
];
962 /* find tb(n) in circular list */
966 tb1
= (TranslationBlock
*)(ntb
& ~3);
967 if (n1
== n
&& tb1
== tb
) {
971 ptb
= &tb1
->jmp_list_first
;
973 ptb
= &tb1
->jmp_list_next
[n1
];
976 /* now we can suppress tb(n) from the list */
977 *ptb
= tb
->jmp_list_next
[n
];
979 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
983 /* reset the jump entry 'n' of a TB so that it is not chained to
985 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
987 uintptr_t addr
= (uintptr_t)(tb
->tc_ptr
+ tb
->jmp_reset_offset
[n
]);
988 tb_set_jmp_target(tb
, n
, addr
);
991 /* remove any jumps to the TB */
992 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
994 TranslationBlock
*tb1
;
998 ptb
= &tb
->jmp_list_first
;
1002 tb1
= (TranslationBlock
*)(ntb
& ~3);
1006 tb_reset_jump(tb1
, n1
);
1007 *ptb
= tb1
->jmp_list_next
[n1
];
1008 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
1012 /* invalidate one TB */
1013 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
1018 tb_page_addr_t phys_pc
;
1020 atomic_set(&tb
->invalid
, true);
1022 /* remove the TB from the hash list */
1023 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1024 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
1025 qht_remove(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1027 /* remove the TB from the page list */
1028 if (tb
->page_addr
[0] != page_addr
) {
1029 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1030 tb_page_remove(&p
->first_tb
, tb
);
1031 invalidate_page_bitmap(p
);
1033 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1034 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1035 tb_page_remove(&p
->first_tb
, tb
);
1036 invalidate_page_bitmap(p
);
1039 /* remove the TB from the hash list */
1040 h
= tb_jmp_cache_hash_func(tb
->pc
);
1042 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1043 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1047 /* suppress this TB from the two jump lists */
1048 tb_remove_from_jmp_list(tb
, 0);
1049 tb_remove_from_jmp_list(tb
, 1);
1051 /* suppress any remaining jumps to this TB */
1054 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1057 #ifdef CONFIG_SOFTMMU
1058 static void build_page_bitmap(PageDesc
*p
)
1060 int n
, tb_start
, tb_end
;
1061 TranslationBlock
*tb
;
1063 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1066 while (tb
!= NULL
) {
1067 n
= (uintptr_t)tb
& 3;
1068 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1069 /* NOTE: this is subtle as a TB may span two physical pages */
1071 /* NOTE: tb_end may be after the end of the page, but
1072 it is not a problem */
1073 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1074 tb_end
= tb_start
+ tb
->size
;
1075 if (tb_end
> TARGET_PAGE_SIZE
) {
1076 tb_end
= TARGET_PAGE_SIZE
;
1080 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1082 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1083 tb
= tb
->page_next
[n
];
1088 /* add the tb in the target page and protect it if necessary
1090 * Called with mmap_lock held for user-mode emulation.
1092 static inline void tb_alloc_page(TranslationBlock
*tb
,
1093 unsigned int n
, tb_page_addr_t page_addr
)
1096 #ifndef CONFIG_USER_ONLY
1097 bool page_already_protected
;
1100 tb
->page_addr
[n
] = page_addr
;
1101 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1102 tb
->page_next
[n
] = p
->first_tb
;
1103 #ifndef CONFIG_USER_ONLY
1104 page_already_protected
= p
->first_tb
!= NULL
;
1106 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1107 invalidate_page_bitmap(p
);
1109 #if defined(CONFIG_USER_ONLY)
1110 if (p
->flags
& PAGE_WRITE
) {
1115 /* force the host page as non writable (writes will have a
1116 page fault + mprotect overhead) */
1117 page_addr
&= qemu_host_page_mask
;
1119 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1120 addr
+= TARGET_PAGE_SIZE
) {
1122 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1127 p2
->flags
&= ~PAGE_WRITE
;
1129 mprotect(g2h(page_addr
), qemu_host_page_size
,
1130 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1131 #ifdef DEBUG_TB_INVALIDATE
1132 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1137 /* if some code is already present, then the pages are already
1138 protected. So we handle the case where only the first TB is
1139 allocated in a physical page */
1140 if (!page_already_protected
) {
1141 tlb_protect_code(page_addr
);
1146 /* add a new TB and link it to the physical page tables. phys_page2 is
1147 * (-1) to indicate that only one page contains the TB.
1149 * Called with mmap_lock held for user-mode emulation.
1151 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1152 tb_page_addr_t phys_page2
)
1156 /* add in the page list */
1157 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1158 if (phys_page2
!= -1) {
1159 tb_alloc_page(tb
, 1, phys_page2
);
1161 tb
->page_addr
[1] = -1;
1164 /* add in the hash table */
1165 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
1166 qht_insert(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1168 #ifdef DEBUG_TB_CHECK
1173 /* Called with mmap_lock held for user mode emulation. */
1174 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1175 target_ulong pc
, target_ulong cs_base
,
1176 uint32_t flags
, int cflags
)
1178 CPUArchState
*env
= cpu
->env_ptr
;
1179 TranslationBlock
*tb
;
1180 tb_page_addr_t phys_pc
, phys_page2
;
1181 target_ulong virt_page2
;
1182 tcg_insn_unit
*gen_code_buf
;
1183 int gen_code_size
, search_size
;
1184 #ifdef CONFIG_PROFILER
1188 phys_pc
= get_page_addr_code(env
, pc
);
1189 if (use_icount
&& !(cflags
& CF_IGNORE_ICOUNT
)) {
1190 cflags
|= CF_USE_ICOUNT
;
1194 if (unlikely(!tb
)) {
1196 /* flush must be done */
1202 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1203 tb
->tc_ptr
= gen_code_buf
;
1204 tb
->cs_base
= cs_base
;
1206 tb
->cflags
= cflags
;
1208 #ifdef CONFIG_PROFILER
1209 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1211 ti
= profile_getclock();
1214 tcg_func_start(&tcg_ctx
);
1216 tcg_ctx
.cpu
= ENV_GET_CPU(env
);
1217 gen_intermediate_code(env
, tb
);
1220 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
1222 /* generate machine code */
1223 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1224 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1225 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1226 #ifdef USE_DIRECT_JUMP
1227 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_insn_offset
;
1228 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1230 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1231 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_addr
;
1234 #ifdef CONFIG_PROFILER
1236 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1237 tcg_ctx
.code_time
-= profile_getclock();
1240 /* ??? Overflow could be handled better here. In particular, we
1241 don't need to re-do gen_intermediate_code, nor should we re-do
1242 the tcg optimization currently hidden inside tcg_gen_code. All
1243 that should be required is to flush the TBs, allocate a new TB,
1244 re-initialize it per above, and re-do the actual code generation. */
1245 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1246 if (unlikely(gen_code_size
< 0)) {
1247 goto buffer_overflow
;
1249 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1250 if (unlikely(search_size
< 0)) {
1251 goto buffer_overflow
;
1254 #ifdef CONFIG_PROFILER
1255 tcg_ctx
.code_time
+= profile_getclock();
1256 tcg_ctx
.code_in_len
+= tb
->size
;
1257 tcg_ctx
.code_out_len
+= gen_code_size
;
1258 tcg_ctx
.search_out_len
+= search_size
;
1262 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1263 qemu_log_in_addr_range(tb
->pc
)) {
1264 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1265 log_disas(tb
->tc_ptr
, gen_code_size
);
1271 tcg_ctx
.code_gen_ptr
= (void *)
1272 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1275 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1276 /* if we are doing vsyscall don't link the page as it lies in high memory
1277 and tb_alloc_page will abort due to page_l1_map returning NULL */
1278 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1279 && phys_pc
< TARGET_VSYSCALL_END
))
1283 /* init jump list */
1284 assert(((uintptr_t)tb
& 3) == 0);
1285 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1286 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1287 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1289 /* init original jump addresses wich has been set during tcg_gen_code() */
1290 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1291 tb_reset_jump(tb
, 0);
1293 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1294 tb_reset_jump(tb
, 1);
1297 /* check next page if needed */
1298 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1300 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1301 phys_page2
= get_page_addr_code(env
, virt_page2
);
1303 /* As long as consistency of the TB stuff is provided by tb_lock in user
1304 * mode and is implicit in single-threaded softmmu emulation, no explicit
1305 * memory barrier is required before tb_link_page() makes the TB visible
1306 * through the physical hash table and physical page list.
1308 tb_link_page(tb
, phys_pc
, phys_page2
);
1313 * Invalidate all TBs which intersect with the target physical address range
1314 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1315 * 'is_cpu_write_access' should be true if called from a real cpu write
1316 * access: the virtual CPU will exit the current TB if code is modified inside
1319 * Called with mmap_lock held for user-mode emulation
1321 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1323 while (start
< end
) {
1324 tb_invalidate_phys_page_range(start
, end
, 0);
1325 start
&= TARGET_PAGE_MASK
;
1326 start
+= TARGET_PAGE_SIZE
;
1331 * Invalidate all TBs which intersect with the target physical address range
1332 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1333 * 'is_cpu_write_access' should be true if called from a real cpu write
1334 * access: the virtual CPU will exit the current TB if code is modified inside
1337 * Called with mmap_lock held for user-mode emulation
1339 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1340 int is_cpu_write_access
)
1342 TranslationBlock
*tb
, *tb_next
;
1343 #if defined(TARGET_HAS_PRECISE_SMC)
1344 CPUState
*cpu
= current_cpu
;
1345 CPUArchState
*env
= NULL
;
1347 tb_page_addr_t tb_start
, tb_end
;
1350 #ifdef TARGET_HAS_PRECISE_SMC
1351 int current_tb_not_found
= is_cpu_write_access
;
1352 TranslationBlock
*current_tb
= NULL
;
1353 int current_tb_modified
= 0;
1354 target_ulong current_pc
= 0;
1355 target_ulong current_cs_base
= 0;
1356 uint32_t current_flags
= 0;
1357 #endif /* TARGET_HAS_PRECISE_SMC */
1359 p
= page_find(start
>> TARGET_PAGE_BITS
);
1363 #if defined(TARGET_HAS_PRECISE_SMC)
1369 /* we remove all the TBs in the range [start, end[ */
1370 /* XXX: see if in some cases it could be faster to invalidate all
1373 while (tb
!= NULL
) {
1374 n
= (uintptr_t)tb
& 3;
1375 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1376 tb_next
= tb
->page_next
[n
];
1377 /* NOTE: this is subtle as a TB may span two physical pages */
1379 /* NOTE: tb_end may be after the end of the page, but
1380 it is not a problem */
1381 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1382 tb_end
= tb_start
+ tb
->size
;
1384 tb_start
= tb
->page_addr
[1];
1385 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1387 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1388 #ifdef TARGET_HAS_PRECISE_SMC
1389 if (current_tb_not_found
) {
1390 current_tb_not_found
= 0;
1392 if (cpu
->mem_io_pc
) {
1393 /* now we have a real cpu fault */
1394 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1397 if (current_tb
== tb
&&
1398 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1399 /* If we are modifying the current TB, we must stop
1400 its execution. We could be more precise by checking
1401 that the modification is after the current PC, but it
1402 would require a specialized function to partially
1403 restore the CPU state */
1405 current_tb_modified
= 1;
1406 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1407 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1410 #endif /* TARGET_HAS_PRECISE_SMC */
1411 tb_phys_invalidate(tb
, -1);
1415 #if !defined(CONFIG_USER_ONLY)
1416 /* if no code remaining, no need to continue to use slow writes */
1418 invalidate_page_bitmap(p
);
1419 tlb_unprotect_code(start
);
1422 #ifdef TARGET_HAS_PRECISE_SMC
1423 if (current_tb_modified
) {
1424 /* we generate a block containing just the instruction
1425 modifying the memory. It will ensure that it cannot modify
1427 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1428 cpu_loop_exit_noexc(cpu
);
1433 #ifdef CONFIG_SOFTMMU
1434 /* len must be <= 8 and start must be a multiple of len */
1435 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1441 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1442 cpu_single_env
->mem_io_vaddr
, len
,
1443 cpu_single_env
->eip
,
1444 cpu_single_env
->eip
+
1445 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1448 p
= page_find(start
>> TARGET_PAGE_BITS
);
1452 if (!p
->code_bitmap
&&
1453 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1454 /* build code bitmap */
1455 build_page_bitmap(p
);
1457 if (p
->code_bitmap
) {
1461 nr
= start
& ~TARGET_PAGE_MASK
;
1462 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1463 if (b
& ((1 << len
) - 1)) {
1468 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1472 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1473 * host PC of the faulting store instruction that caused this invalidate.
1474 * Returns true if the caller needs to abort execution of the current
1475 * TB (because it was modified by this store and the guest CPU has
1476 * precise-SMC semantics).
1478 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
1480 TranslationBlock
*tb
;
1483 #ifdef TARGET_HAS_PRECISE_SMC
1484 TranslationBlock
*current_tb
= NULL
;
1485 CPUState
*cpu
= current_cpu
;
1486 CPUArchState
*env
= NULL
;
1487 int current_tb_modified
= 0;
1488 target_ulong current_pc
= 0;
1489 target_ulong current_cs_base
= 0;
1490 uint32_t current_flags
= 0;
1493 addr
&= TARGET_PAGE_MASK
;
1494 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1499 #ifdef TARGET_HAS_PRECISE_SMC
1500 if (tb
&& pc
!= 0) {
1501 current_tb
= tb_find_pc(pc
);
1507 while (tb
!= NULL
) {
1508 n
= (uintptr_t)tb
& 3;
1509 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1510 #ifdef TARGET_HAS_PRECISE_SMC
1511 if (current_tb
== tb
&&
1512 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1513 /* If we are modifying the current TB, we must stop
1514 its execution. We could be more precise by checking
1515 that the modification is after the current PC, but it
1516 would require a specialized function to partially
1517 restore the CPU state */
1519 current_tb_modified
= 1;
1520 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1521 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1524 #endif /* TARGET_HAS_PRECISE_SMC */
1525 tb_phys_invalidate(tb
, addr
);
1526 tb
= tb
->page_next
[n
];
1529 #ifdef TARGET_HAS_PRECISE_SMC
1530 if (current_tb_modified
) {
1531 /* we generate a block containing just the instruction
1532 modifying the memory. It will ensure that it cannot modify
1534 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1542 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1543 tb[1].tc_ptr. Return NULL if not found */
1544 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1546 int m_min
, m_max
, m
;
1548 TranslationBlock
*tb
;
1550 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1553 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1554 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1557 /* binary search (cf Knuth) */
1559 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1560 while (m_min
<= m_max
) {
1561 m
= (m_min
+ m_max
) >> 1;
1562 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1563 v
= (uintptr_t)tb
->tc_ptr
;
1566 } else if (tc_ptr
< v
) {
1572 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1575 #if !defined(CONFIG_USER_ONLY)
1576 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1578 ram_addr_t ram_addr
;
1583 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1584 if (!(memory_region_is_ram(mr
)
1585 || memory_region_is_romd(mr
))) {
1589 ram_addr
= memory_region_get_ram_addr(mr
) + addr
;
1590 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1593 #endif /* !defined(CONFIG_USER_ONLY) */
1595 void tb_check_watchpoint(CPUState
*cpu
)
1597 TranslationBlock
*tb
;
1599 tb
= tb_find_pc(cpu
->mem_io_pc
);
1601 /* We can use retranslation to find the PC. */
1602 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1603 tb_phys_invalidate(tb
, -1);
1605 /* The exception probably happened in a helper. The CPU state should
1606 have been saved before calling it. Fetch the PC from there. */
1607 CPUArchState
*env
= cpu
->env_ptr
;
1608 target_ulong pc
, cs_base
;
1609 tb_page_addr_t addr
;
1612 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1613 addr
= get_page_addr_code(env
, pc
);
1614 tb_invalidate_phys_range(addr
, addr
+ 1);
1618 #ifndef CONFIG_USER_ONLY
1619 /* in deterministic execution mode, instructions doing device I/Os
1620 must be at the end of the TB */
1621 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1623 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1624 CPUArchState
*env
= cpu
->env_ptr
;
1626 TranslationBlock
*tb
;
1628 target_ulong pc
, cs_base
;
1631 tb
= tb_find_pc(retaddr
);
1633 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1636 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1637 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1638 /* Calculate how many instructions had been executed before the fault
1640 n
= n
- cpu
->icount_decr
.u16
.low
;
1641 /* Generate a new TB ending on the I/O insn. */
1643 /* On MIPS and SH, delay slot instructions can only be restarted if
1644 they were already the first instruction in the TB. If this is not
1645 the first instruction in a TB then re-execute the preceding
1647 #if defined(TARGET_MIPS)
1648 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1649 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1650 cpu
->icount_decr
.u16
.low
++;
1651 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1653 #elif defined(TARGET_SH4)
1654 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1657 cpu
->icount_decr
.u16
.low
++;
1658 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1661 /* This should never happen. */
1662 if (n
> CF_COUNT_MASK
) {
1663 cpu_abort(cpu
, "TB too big during recompile");
1666 cflags
= n
| CF_LAST_IO
;
1668 cs_base
= tb
->cs_base
;
1670 tb_phys_invalidate(tb
, -1);
1671 if (tb
->cflags
& CF_NOCACHE
) {
1673 /* Invalidate original TB if this TB was generated in
1674 * cpu_exec_nocache() */
1675 tb_phys_invalidate(tb
->orig_tb
, -1);
1679 /* FIXME: In theory this could raise an exception. In practice
1680 we have already translated the block once so it's probably ok. */
1681 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1682 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1683 the first in the TB) then we end up generating a whole new TB and
1684 repeating the fault, which is horribly inefficient.
1685 Better would be to execute just this insn uncached, or generate a
1687 cpu_loop_exit_noexc(cpu
);
1690 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1694 /* Discard jump cache entries for any tb which might potentially
1695 overlap the flushed page. */
1696 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1697 memset(&cpu
->tb_jmp_cache
[i
], 0,
1698 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1700 i
= tb_jmp_cache_hash_page(addr
);
1701 memset(&cpu
->tb_jmp_cache
[i
], 0,
1702 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1705 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
1706 struct qht_stats hst
)
1708 uint32_t hgram_opts
;
1712 if (!hst
.head_buckets
) {
1715 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1716 hst
.used_head_buckets
, hst
.head_buckets
,
1717 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
1719 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1720 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
1721 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
1722 hgram_opts
|= QDIST_PR_NODECIMAL
;
1724 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
1725 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1726 qdist_avg(&hst
.occupancy
) * 100, hgram
);
1729 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1730 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
1731 if (hgram_bins
> 10) {
1735 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
1737 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
1738 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1739 qdist_avg(&hst
.chain
), hgram
);
1743 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1745 int i
, target_code_size
, max_target_code_size
;
1746 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1747 TranslationBlock
*tb
;
1748 struct qht_stats hst
;
1750 target_code_size
= 0;
1751 max_target_code_size
= 0;
1753 direct_jmp_count
= 0;
1754 direct_jmp2_count
= 0;
1755 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1756 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1757 target_code_size
+= tb
->size
;
1758 if (tb
->size
> max_target_code_size
) {
1759 max_target_code_size
= tb
->size
;
1761 if (tb
->page_addr
[1] != -1) {
1764 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1766 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1767 direct_jmp2_count
++;
1771 /* XXX: avoid using doubles ? */
1772 cpu_fprintf(f
, "Translation buffer state:\n");
1773 cpu_fprintf(f
, "gen code size %td/%zd\n",
1774 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1775 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1776 cpu_fprintf(f
, "TB count %d/%d\n",
1777 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1778 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1779 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1780 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1781 max_target_code_size
);
1782 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1783 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1784 tcg_ctx
.code_gen_buffer
) /
1785 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1786 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1787 tcg_ctx
.code_gen_buffer
) /
1788 target_code_size
: 0);
1789 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1790 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1791 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1792 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1794 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1795 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1797 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1798 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1800 qht_statistics_init(&tcg_ctx
.tb_ctx
.htable
, &hst
);
1801 print_qht_statistics(f
, cpu_fprintf
, hst
);
1802 qht_statistics_destroy(&hst
);
1804 cpu_fprintf(f
, "\nStatistics:\n");
1805 cpu_fprintf(f
, "TB flush count %u\n",
1806 atomic_read(&tcg_ctx
.tb_ctx
.tb_flush_count
));
1807 cpu_fprintf(f
, "TB invalidate count %d\n",
1808 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1809 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1810 tcg_dump_info(f
, cpu_fprintf
);
1813 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1815 tcg_dump_op_count(f
, cpu_fprintf
);
1818 #else /* CONFIG_USER_ONLY */
1820 void cpu_interrupt(CPUState
*cpu
, int mask
)
1822 cpu
->interrupt_request
|= mask
;
1823 cpu
->tcg_exit_req
= 1;
1827 * Walks guest process memory "regions" one by one
1828 * and calls callback function 'fn' for each region.
1830 struct walk_memory_regions_data
{
1831 walk_memory_regions_fn fn
;
1837 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1838 target_ulong end
, int new_prot
)
1840 if (data
->start
!= -1u) {
1841 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1847 data
->start
= (new_prot
? end
: -1u);
1848 data
->prot
= new_prot
;
1853 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1854 target_ulong base
, int level
, void **lp
)
1860 return walk_memory_regions_end(data
, base
, 0);
1866 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1867 int prot
= pd
[i
].flags
;
1869 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1870 if (prot
!= data
->prot
) {
1871 rc
= walk_memory_regions_end(data
, pa
, prot
);
1880 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1881 pa
= base
| ((target_ulong
)i
<<
1882 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1883 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1893 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1895 struct walk_memory_regions_data data
;
1903 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1904 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1905 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1911 return walk_memory_regions_end(&data
, 0, 0);
1914 static int dump_region(void *priv
, target_ulong start
,
1915 target_ulong end
, abi_ulong prot
)
1917 FILE *f
= (FILE *)priv
;
1919 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1920 " "TARGET_FMT_lx
" %c%c%c\n",
1921 start
, end
, end
- start
,
1922 ((prot
& PAGE_READ
) ? 'r' : '-'),
1923 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1924 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1929 /* dump memory mappings */
1930 void page_dump(FILE *f
)
1932 const int length
= sizeof(target_ulong
) * 2;
1933 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1934 length
, "start", length
, "end", length
, "size", "prot");
1935 walk_memory_regions(f
, dump_region
);
1938 int page_get_flags(target_ulong address
)
1942 p
= page_find(address
>> TARGET_PAGE_BITS
);
1949 /* Modify the flags of a page and invalidate the code if necessary.
1950 The flag PAGE_WRITE_ORG is positioned automatically depending
1951 on PAGE_WRITE. The mmap_lock should already be held. */
1952 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1954 target_ulong addr
, len
;
1956 /* This function should never be called with addresses outside the
1957 guest address space. If this assert fires, it probably indicates
1958 a missing call to h2g_valid. */
1959 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1960 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1962 assert(start
< end
);
1964 start
= start
& TARGET_PAGE_MASK
;
1965 end
= TARGET_PAGE_ALIGN(end
);
1967 if (flags
& PAGE_WRITE
) {
1968 flags
|= PAGE_WRITE_ORG
;
1971 for (addr
= start
, len
= end
- start
;
1973 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1974 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1976 /* If the write protection bit is set, then we invalidate
1978 if (!(p
->flags
& PAGE_WRITE
) &&
1979 (flags
& PAGE_WRITE
) &&
1981 tb_invalidate_phys_page(addr
, 0);
1987 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1993 /* This function should never be called with addresses outside the
1994 guest address space. If this assert fires, it probably indicates
1995 a missing call to h2g_valid. */
1996 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1997 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2003 if (start
+ len
- 1 < start
) {
2004 /* We've wrapped around. */
2008 /* must do before we loose bits in the next step */
2009 end
= TARGET_PAGE_ALIGN(start
+ len
);
2010 start
= start
& TARGET_PAGE_MASK
;
2012 for (addr
= start
, len
= end
- start
;
2014 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2015 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2019 if (!(p
->flags
& PAGE_VALID
)) {
2023 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
2026 if (flags
& PAGE_WRITE
) {
2027 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2030 /* unprotect the page if it was put read-only because it
2031 contains translated code */
2032 if (!(p
->flags
& PAGE_WRITE
)) {
2033 if (!page_unprotect(addr
, 0)) {
2042 /* called from signal handler: invalidate the code and unprotect the
2043 * page. Return 0 if the fault was not handled, 1 if it was handled,
2044 * and 2 if it was handled but the caller must cause the TB to be
2045 * immediately exited. (We can only return 2 if the 'pc' argument is
2048 int page_unprotect(target_ulong address
, uintptr_t pc
)
2051 bool current_tb_invalidated
;
2053 target_ulong host_start
, host_end
, addr
;
2055 /* Technically this isn't safe inside a signal handler. However we
2056 know this only ever happens in a synchronous SEGV handler, so in
2057 practice it seems to be ok. */
2060 p
= page_find(address
>> TARGET_PAGE_BITS
);
2066 /* if the page was really writable, then we change its
2067 protection back to writable */
2068 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2069 host_start
= address
& qemu_host_page_mask
;
2070 host_end
= host_start
+ qemu_host_page_size
;
2073 current_tb_invalidated
= false;
2074 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2075 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2076 p
->flags
|= PAGE_WRITE
;
2079 /* and since the content will be modified, we must invalidate
2080 the corresponding translated code. */
2081 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2082 #ifdef DEBUG_TB_CHECK
2083 tb_invalidate_check(addr
);
2086 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2090 /* If current TB was invalidated return to main loop */
2091 return current_tb_invalidated
? 2 : 1;
2096 #endif /* CONFIG_USER_ONLY */