4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "qemu/osdep.h"
24 #include "qemu-common.h"
25 #define NO_CPU_IO_DEFS
28 #include "disas/disas.h"
29 #include "exec/exec-all.h"
31 #if defined(CONFIG_USER_ONLY)
33 #if defined(TARGET_X86_64)
36 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
37 #include <sys/param.h>
38 #if __FreeBSD_version >= 700104
39 #define HAVE_KINFO_GETVMMAP
40 #define sigqueue sigqueue_freebsd /* avoid redefinition */
42 #include <machine/profile.h>
51 #include "exec/address-spaces.h"
54 #include "exec/cputlb.h"
55 #include "exec/tb-hash.h"
56 #include "translate-all.h"
57 #include "qemu/bitmap.h"
58 #include "qemu/error-report.h"
59 #include "qemu/timer.h"
60 #include "qemu/main-loop.h"
62 #include "sysemu/cpus.h"
64 /* #define DEBUG_TB_INVALIDATE */
65 /* #define DEBUG_TB_FLUSH */
66 /* make various TB consistency checks */
67 /* #define DEBUG_TB_CHECK */
69 #ifdef DEBUG_TB_INVALIDATE
70 #define DEBUG_TB_INVALIDATE_GATE 1
72 #define DEBUG_TB_INVALIDATE_GATE 0
76 #define DEBUG_TB_FLUSH_GATE 1
78 #define DEBUG_TB_FLUSH_GATE 0
81 #if !defined(CONFIG_USER_ONLY)
82 /* TB consistency checks only implemented for usermode emulation. */
87 #define DEBUG_TB_CHECK_GATE 1
89 #define DEBUG_TB_CHECK_GATE 0
92 /* Access to the various translations structures need to be serialised via locks
93 * for consistency. This is automatic for SoftMMU based system
94 * emulation due to its single threaded nature. In user-mode emulation
95 * access to the memory related structures are protected with the
99 #define assert_memory_lock() tcg_debug_assert(have_tb_lock)
101 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
104 #define SMC_BITMAP_USE_THRESHOLD 10
106 typedef struct PageDesc
{
107 /* list of TBs intersecting this ram page */
108 TranslationBlock
*first_tb
;
109 #ifdef CONFIG_SOFTMMU
110 /* in order to optimize self modifying code, we count the number
111 of lookups we do to a given page to use a bitmap */
112 unsigned int code_write_count
;
113 unsigned long *code_bitmap
;
119 /* In system mode we want L1_MAP to be based on ram offsets,
120 while in user mode we want it to be based on virtual addresses. */
121 #if !defined(CONFIG_USER_ONLY)
122 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
123 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
125 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
128 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
131 /* Size of the L2 (and L3, etc) page tables. */
133 #define V_L2_SIZE (1 << V_L2_BITS)
135 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
136 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS
>
137 sizeof(((TranslationBlock
*)0)->trace_vcpu_dstate
)
141 * L1 Mapping properties
143 static int v_l1_size
;
144 static int v_l1_shift
;
145 static int v_l2_levels
;
147 /* The bottom level has pointers to PageDesc, and is indexed by
148 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
150 #define V_L1_MIN_BITS 4
151 #define V_L1_MAX_BITS (V_L2_BITS + 3)
152 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
154 static void *l1_map
[V_L1_MAX_SIZE
];
156 /* code generation context */
157 TCGContext tcg_init_ctx
;
158 __thread TCGContext
*tcg_ctx
;
162 /* translation block context */
163 static __thread
int have_tb_lock
;
165 static void page_table_config_init(void)
169 assert(TARGET_PAGE_BITS
);
170 /* The bits remaining after N lower levels of page tables. */
171 v_l1_bits
= (L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
) % V_L2_BITS
;
172 if (v_l1_bits
< V_L1_MIN_BITS
) {
173 v_l1_bits
+= V_L2_BITS
;
176 v_l1_size
= 1 << v_l1_bits
;
177 v_l1_shift
= L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
- v_l1_bits
;
178 v_l2_levels
= v_l1_shift
/ V_L2_BITS
- 1;
180 assert(v_l1_bits
<= V_L1_MAX_BITS
);
181 assert(v_l1_shift
% V_L2_BITS
== 0);
182 assert(v_l2_levels
>= 0);
185 #define assert_tb_locked() tcg_debug_assert(have_tb_lock)
186 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock)
190 assert_tb_unlocked();
191 qemu_mutex_lock(&tb_ctx
.tb_lock
);
199 qemu_mutex_unlock(&tb_ctx
.tb_lock
);
202 void tb_lock_reset(void)
205 qemu_mutex_unlock(&tb_ctx
.tb_lock
);
210 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
212 void cpu_gen_init(void)
214 tcg_context_init(&tcg_init_ctx
);
217 /* Encode VAL as a signed leb128 sequence at P.
218 Return P incremented past the encoded value. */
219 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
226 more
= !((val
== 0 && (byte
& 0x40) == 0)
227 || (val
== -1 && (byte
& 0x40) != 0));
237 /* Decode a signed leb128 sequence at *PP; increment *PP past the
238 decoded value. Return the decoded value. */
239 static target_long
decode_sleb128(uint8_t **pp
)
247 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
249 } while (byte
& 0x80);
250 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
251 val
|= -(target_ulong
)1 << shift
;
258 /* Encode the data collected about the instructions while compiling TB.
259 Place the data at BLOCK, and return the number of bytes consumed.
261 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
262 which come from the target's insn_start data, followed by a uintptr_t
263 which comes from the host pc of the end of the code implementing the insn.
265 Each line of the table is encoded as sleb128 deltas from the previous
266 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
267 That is, the first column is seeded with the guest pc, the last column
268 with the host pc, and the middle columns with zeros. */
270 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
272 uint8_t *highwater
= tcg_ctx
->code_gen_highwater
;
276 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
279 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
281 prev
= (j
== 0 ? tb
->pc
: 0);
283 prev
= tcg_ctx
->gen_insn_data
[i
- 1][j
];
285 p
= encode_sleb128(p
, tcg_ctx
->gen_insn_data
[i
][j
] - prev
);
287 prev
= (i
== 0 ? 0 : tcg_ctx
->gen_insn_end_off
[i
- 1]);
288 p
= encode_sleb128(p
, tcg_ctx
->gen_insn_end_off
[i
] - prev
);
290 /* Test for (pending) buffer overflow. The assumption is that any
291 one row beginning below the high water mark cannot overrun
292 the buffer completely. Thus we can test for overflow after
293 encoding a row without having to check during encoding. */
294 if (unlikely(p
> highwater
)) {
302 /* The cpu state corresponding to 'searched_pc' is restored.
303 * Called with tb_lock held.
305 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
306 uintptr_t searched_pc
)
308 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
309 uintptr_t host_pc
= (uintptr_t)tb
->tc
.ptr
;
310 CPUArchState
*env
= cpu
->env_ptr
;
311 uint8_t *p
= tb
->tc
.ptr
+ tb
->tc
.size
;
312 int i
, j
, num_insns
= tb
->icount
;
313 #ifdef CONFIG_PROFILER
314 TCGProfile
*prof
= &tcg_ctx
->prof
;
315 int64_t ti
= profile_getclock();
318 searched_pc
-= GETPC_ADJ
;
320 if (searched_pc
< host_pc
) {
324 /* Reconstruct the stored insn data while looking for the point at
325 which the end of the insn exceeds the searched_pc. */
326 for (i
= 0; i
< num_insns
; ++i
) {
327 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
328 data
[j
] += decode_sleb128(&p
);
330 host_pc
+= decode_sleb128(&p
);
331 if (host_pc
> searched_pc
) {
338 if (tb
->cflags
& CF_USE_ICOUNT
) {
340 /* Reset the cycle counter to the start of the block. */
341 cpu
->icount_decr
.u16
.low
+= num_insns
;
342 /* Clear the IO flag. */
345 cpu
->icount_decr
.u16
.low
-= i
;
346 restore_state_to_opc(env
, tb
, data
);
348 #ifdef CONFIG_PROFILER
349 atomic_set(&prof
->restore_time
,
350 prof
->restore_time
+ profile_getclock() - ti
);
351 atomic_set(&prof
->restore_count
, prof
->restore_count
+ 1);
356 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
358 TranslationBlock
*tb
;
361 /* A retaddr of zero is invalid so we really shouldn't have ended
362 * up here. The target code has likely forgotten to check retaddr
363 * != 0 before attempting to restore state. We return early to
364 * avoid blowing up on a recursive tb_lock(). The target must have
365 * previously survived a failed cpu_restore_state because
366 * tb_find_pc(0) would have failed anyway. It still should be
375 tb
= tb_find_pc(retaddr
);
377 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
378 if (tb
->cflags
& CF_NOCACHE
) {
379 /* one-shot translation, invalidate it immediately */
380 tb_phys_invalidate(tb
, -1);
390 static void page_init(void)
393 page_table_config_init();
395 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
397 #ifdef HAVE_KINFO_GETVMMAP
398 struct kinfo_vmentry
*freep
;
401 freep
= kinfo_getvmmap(getpid(), &cnt
);
404 for (i
= 0; i
< cnt
; i
++) {
405 unsigned long startaddr
, endaddr
;
407 startaddr
= freep
[i
].kve_start
;
408 endaddr
= freep
[i
].kve_end
;
409 if (h2g_valid(startaddr
)) {
410 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
412 if (h2g_valid(endaddr
)) {
413 endaddr
= h2g(endaddr
);
414 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
416 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
418 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
429 last_brk
= (unsigned long)sbrk(0);
431 f
= fopen("/compat/linux/proc/self/maps", "r");
436 unsigned long startaddr
, endaddr
;
439 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
441 if (n
== 2 && h2g_valid(startaddr
)) {
442 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
444 if (h2g_valid(endaddr
)) {
445 endaddr
= h2g(endaddr
);
449 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
462 * Called with tb_lock held for system emulation.
463 * Called with mmap_lock held for user-mode emulation.
465 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
472 assert_memory_lock();
475 /* Level 1. Always allocated. */
476 lp
= l1_map
+ ((index
>> v_l1_shift
) & (v_l1_size
- 1));
479 for (i
= v_l2_levels
; i
> 0; i
--) {
480 void **p
= atomic_rcu_read(lp
);
486 p
= g_new0(void *, V_L2_SIZE
);
487 atomic_rcu_set(lp
, p
);
490 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
493 pd
= atomic_rcu_read(lp
);
498 pd
= g_new0(PageDesc
, V_L2_SIZE
);
499 atomic_rcu_set(lp
, pd
);
502 return pd
+ (index
& (V_L2_SIZE
- 1));
505 static inline PageDesc
*page_find(tb_page_addr_t index
)
507 return page_find_alloc(index
, 0);
510 #if defined(CONFIG_USER_ONLY)
511 /* Currently it is not recommended to allocate big chunks of data in
512 user mode. It will change when a dedicated libc will be used. */
513 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
514 region in which the guest needs to run. Revisit this. */
515 #define USE_STATIC_CODE_GEN_BUFFER
518 /* Minimum size of the code gen buffer. This number is randomly chosen,
519 but not so small that we can't have a fair number of TB's live. */
520 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
522 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
523 indicated, this is constrained by the range of direct branches on the
524 host cpu, as used by the TCG implementation of goto_tb. */
525 #if defined(__x86_64__)
526 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
527 #elif defined(__sparc__)
528 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
529 #elif defined(__powerpc64__)
530 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
531 #elif defined(__powerpc__)
532 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
533 #elif defined(__aarch64__)
534 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
535 #elif defined(__s390x__)
536 /* We have a +- 4GB range on the branches; leave some slop. */
537 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
538 #elif defined(__mips__)
539 /* We have a 256MB branch region, but leave room to make sure the
540 main executable is also within that region. */
541 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
543 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
546 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
548 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
549 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
550 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
552 static inline size_t size_code_gen_buffer(size_t tb_size
)
554 /* Size the buffer. */
556 #ifdef USE_STATIC_CODE_GEN_BUFFER
557 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
559 /* ??? Needs adjustments. */
560 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
561 static buffer, we could size this on RESERVED_VA, on the text
562 segment size of the executable, or continue to use the default. */
563 tb_size
= (unsigned long)(ram_size
/ 4);
566 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
567 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
569 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
570 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
576 /* In order to use J and JAL within the code_gen_buffer, we require
577 that the buffer not cross a 256MB boundary. */
578 static inline bool cross_256mb(void *addr
, size_t size
)
580 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
583 /* We weren't able to allocate a buffer without crossing that boundary,
584 so make do with the larger portion of the buffer that doesn't cross.
585 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
586 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
588 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
589 size_t size2
= buf1
+ size1
- buf2
;
597 tcg_ctx
->code_gen_buffer_size
= size1
;
602 #ifdef USE_STATIC_CODE_GEN_BUFFER
603 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
604 __attribute__((aligned(CODE_GEN_ALIGN
)));
606 static inline void *alloc_code_gen_buffer(void)
608 void *buf
= static_code_gen_buffer
;
609 void *end
= static_code_gen_buffer
+ sizeof(static_code_gen_buffer
);
612 /* page-align the beginning and end of the buffer */
613 buf
= QEMU_ALIGN_PTR_UP(buf
, qemu_real_host_page_size
);
614 end
= QEMU_ALIGN_PTR_DOWN(end
, qemu_real_host_page_size
);
618 /* Honor a command-line option limiting the size of the buffer. */
619 if (size
> tcg_ctx
->code_gen_buffer_size
) {
620 size
= QEMU_ALIGN_DOWN(tcg_ctx
->code_gen_buffer_size
,
621 qemu_real_host_page_size
);
623 tcg_ctx
->code_gen_buffer_size
= size
;
626 if (cross_256mb(buf
, size
)) {
627 buf
= split_cross_256mb(buf
, size
);
628 size
= tcg_ctx
->code_gen_buffer_size
;
632 if (qemu_mprotect_rwx(buf
, size
)) {
635 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
639 #elif defined(_WIN32)
640 static inline void *alloc_code_gen_buffer(void)
642 size_t size
= tcg_ctx
->code_gen_buffer_size
;
645 buf
= VirtualAlloc(NULL
, size
, MEM_RESERVE
| MEM_COMMIT
,
646 PAGE_EXECUTE_READWRITE
);
650 static inline void *alloc_code_gen_buffer(void)
652 int prot
= PROT_WRITE
| PROT_READ
| PROT_EXEC
;
653 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
655 size_t size
= tcg_ctx
->code_gen_buffer_size
;
658 /* Constrain the position of the buffer based on the host cpu.
659 Note that these addresses are chosen in concert with the
660 addresses assigned in the relevant linker script file. */
661 # if defined(__PIE__) || defined(__PIC__)
662 /* Don't bother setting a preferred location if we're building
663 a position-independent executable. We're more likely to get
664 an address near the main executable if we let the kernel
665 choose the address. */
666 # elif defined(__x86_64__) && defined(MAP_32BIT)
667 /* Force the memory down into low memory with the executable.
668 Leave the choice of exact location with the kernel. */
670 /* Cannot expect to map more than 800MB in low memory. */
671 if (size
> 800u * 1024 * 1024) {
672 tcg_ctx
->code_gen_buffer_size
= size
= 800u * 1024 * 1024;
674 # elif defined(__sparc__)
675 start
= 0x40000000ul
;
676 # elif defined(__s390x__)
677 start
= 0x90000000ul
;
678 # elif defined(__mips__)
679 # if _MIPS_SIM == _ABI64
680 start
= 0x128000000ul
;
682 start
= 0x08000000ul
;
686 buf
= mmap((void *)start
, size
, prot
, flags
, -1, 0);
687 if (buf
== MAP_FAILED
) {
692 if (cross_256mb(buf
, size
)) {
693 /* Try again, with the original still mapped, to avoid re-acquiring
694 that 256mb crossing. This time don't specify an address. */
696 void *buf2
= mmap(NULL
, size
, prot
, flags
, -1, 0);
697 switch ((int)(buf2
!= MAP_FAILED
)) {
699 if (!cross_256mb(buf2
, size
)) {
700 /* Success! Use the new buffer. */
704 /* Failure. Work with what we had. */
708 /* Split the original buffer. Free the smaller half. */
709 buf2
= split_cross_256mb(buf
, size
);
710 size2
= tcg_ctx
->code_gen_buffer_size
;
712 munmap(buf
+ size2
, size
- size2
);
714 munmap(buf
, size
- size2
);
723 /* Request large pages for the buffer. */
724 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
728 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
730 /* compare a pointer @ptr and a tb_tc @s */
731 static int ptr_cmp_tb_tc(const void *ptr
, const struct tb_tc
*s
)
733 if (ptr
>= s
->ptr
+ s
->size
) {
735 } else if (ptr
< s
->ptr
) {
741 static gint
tb_tc_cmp(gconstpointer ap
, gconstpointer bp
)
743 const struct tb_tc
*a
= ap
;
744 const struct tb_tc
*b
= bp
;
747 * When both sizes are set, we know this isn't a lookup.
748 * This is the most likely case: every TB must be inserted; lookups
749 * are a lot less frequent.
751 if (likely(a
->size
&& b
->size
)) {
752 if (a
->ptr
> b
->ptr
) {
754 } else if (a
->ptr
< b
->ptr
) {
757 /* a->ptr == b->ptr should happen only on deletions */
758 g_assert(a
->size
== b
->size
);
762 * All lookups have either .size field set to 0.
763 * From the glib sources we see that @ap is always the lookup key. However
764 * the docs provide no guarantee, so we just mark this case as likely.
766 if (likely(a
->size
== 0)) {
767 return ptr_cmp_tb_tc(a
->ptr
, b
);
769 return ptr_cmp_tb_tc(b
->ptr
, a
);
772 static inline void code_gen_alloc(size_t tb_size
)
774 tcg_ctx
->code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
775 tcg_ctx
->code_gen_buffer
= alloc_code_gen_buffer();
776 if (tcg_ctx
->code_gen_buffer
== NULL
) {
777 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
780 tb_ctx
.tb_tree
= g_tree_new(tb_tc_cmp
);
781 qemu_mutex_init(&tb_ctx
.tb_lock
);
784 static void tb_htable_init(void)
786 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
788 qht_init(&tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
, mode
);
791 /* Must be called before using the QEMU cpus. 'tb_size' is the size
792 (in bytes) allocated to the translation buffer. Zero means default
794 void tcg_exec_init(uintptr_t tb_size
)
800 code_gen_alloc(tb_size
);
801 #if defined(CONFIG_SOFTMMU)
802 /* There's no guest base to take into account, so go ahead and
803 initialize the prologue now. */
804 tcg_prologue_init(tcg_ctx
);
809 * Allocate a new translation block. Flush the translation buffer if
810 * too many translation blocks or too much generated code.
812 * Called with tb_lock held.
814 static TranslationBlock
*tb_alloc(target_ulong pc
)
816 TranslationBlock
*tb
;
820 tb
= tcg_tb_alloc(tcg_ctx
);
821 if (unlikely(tb
== NULL
)) {
827 /* Called with tb_lock held. */
828 void tb_remove(TranslationBlock
*tb
)
832 g_tree_remove(tb_ctx
.tb_tree
, &tb
->tc
);
835 static inline void invalidate_page_bitmap(PageDesc
*p
)
837 #ifdef CONFIG_SOFTMMU
838 g_free(p
->code_bitmap
);
839 p
->code_bitmap
= NULL
;
840 p
->code_write_count
= 0;
844 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
845 static void page_flush_tb_1(int level
, void **lp
)
855 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
856 pd
[i
].first_tb
= NULL
;
857 invalidate_page_bitmap(pd
+ i
);
862 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
863 page_flush_tb_1(level
- 1, pp
+ i
);
868 static void page_flush_tb(void)
870 int i
, l1_sz
= v_l1_size
;
872 for (i
= 0; i
< l1_sz
; i
++) {
873 page_flush_tb_1(v_l2_levels
, l1_map
+ i
);
877 static gboolean
tb_host_size_iter(gpointer key
, gpointer value
, gpointer data
)
879 const TranslationBlock
*tb
= value
;
882 *size
+= tb
->tc
.size
;
886 /* flush all the translation blocks */
887 static void do_tb_flush(CPUState
*cpu
, run_on_cpu_data tb_flush_count
)
891 /* If it is already been done on request of another CPU,
894 if (tb_ctx
.tb_flush_count
!= tb_flush_count
.host_int
) {
898 if (DEBUG_TB_FLUSH_GATE
) {
899 size_t nb_tbs
= g_tree_nnodes(tb_ctx
.tb_tree
);
900 size_t host_size
= 0;
902 g_tree_foreach(tb_ctx
.tb_tree
, tb_host_size_iter
, &host_size
);
903 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
904 tcg_code_size(), nb_tbs
, nb_tbs
> 0 ? host_size
/ nb_tbs
: 0);
908 cpu_tb_jmp_cache_clear(cpu
);
911 /* Increment the refcount first so that destroy acts as a reset */
912 g_tree_ref(tb_ctx
.tb_tree
);
913 g_tree_destroy(tb_ctx
.tb_tree
);
915 qht_reset_size(&tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
918 tcg_region_reset_all();
919 /* XXX: flush processor icache at this point if cache flush is
921 atomic_mb_set(&tb_ctx
.tb_flush_count
, tb_ctx
.tb_flush_count
+ 1);
927 void tb_flush(CPUState
*cpu
)
930 unsigned tb_flush_count
= atomic_mb_read(&tb_ctx
.tb_flush_count
);
931 async_safe_run_on_cpu(cpu
, do_tb_flush
,
932 RUN_ON_CPU_HOST_INT(tb_flush_count
));
937 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
938 * so in order to prevent bit rot we compile them unconditionally in user-mode,
939 * and let the optimizer get rid of them by wrapping their user-only callers
940 * with if (DEBUG_TB_CHECK_GATE).
942 #ifdef CONFIG_USER_ONLY
945 do_tb_invalidate_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
947 TranslationBlock
*tb
= p
;
948 target_ulong addr
= *(target_ulong
*)userp
;
950 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
951 printf("ERROR invalidate: address=" TARGET_FMT_lx
952 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
956 /* verify that all the pages have correct rights for code
958 * Called with tb_lock held.
960 static void tb_invalidate_check(target_ulong address
)
962 address
&= TARGET_PAGE_MASK
;
963 qht_iter(&tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
967 do_tb_page_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
969 TranslationBlock
*tb
= p
;
972 flags1
= page_get_flags(tb
->pc
);
973 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
974 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
975 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
976 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
980 /* verify that all the pages have correct rights for code */
981 static void tb_page_check(void)
983 qht_iter(&tb_ctx
.htable
, do_tb_page_check
, NULL
);
986 #endif /* CONFIG_USER_ONLY */
988 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
990 TranslationBlock
*tb1
;
995 n1
= (uintptr_t)tb1
& 3;
996 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
998 *ptb
= tb1
->page_next
[n1
];
1001 ptb
= &tb1
->page_next
[n1
];
1005 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
1006 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
1008 TranslationBlock
*tb1
;
1009 uintptr_t *ptb
, ntb
;
1012 ptb
= &tb
->jmp_list_next
[n
];
1014 /* find tb(n) in circular list */
1018 tb1
= (TranslationBlock
*)(ntb
& ~3);
1019 if (n1
== n
&& tb1
== tb
) {
1023 ptb
= &tb1
->jmp_list_first
;
1025 ptb
= &tb1
->jmp_list_next
[n1
];
1028 /* now we can suppress tb(n) from the list */
1029 *ptb
= tb
->jmp_list_next
[n
];
1031 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
1035 /* reset the jump entry 'n' of a TB so that it is not chained to
1037 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
1039 uintptr_t addr
= (uintptr_t)(tb
->tc
.ptr
+ tb
->jmp_reset_offset
[n
]);
1040 tb_set_jmp_target(tb
, n
, addr
);
1043 /* remove any jumps to the TB */
1044 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
1046 TranslationBlock
*tb1
;
1047 uintptr_t *ptb
, ntb
;
1050 ptb
= &tb
->jmp_list_first
;
1054 tb1
= (TranslationBlock
*)(ntb
& ~3);
1058 tb_reset_jump(tb1
, n1
);
1059 *ptb
= tb1
->jmp_list_next
[n1
];
1060 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
1064 /* invalidate one TB
1066 * Called with tb_lock held.
1068 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
1073 tb_page_addr_t phys_pc
;
1077 atomic_set(&tb
->cflags
, tb
->cflags
| CF_INVALID
);
1079 /* remove the TB from the hash list */
1080 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1081 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->cflags
& CF_HASH_MASK
,
1082 tb
->trace_vcpu_dstate
);
1083 if (!qht_remove(&tb_ctx
.htable
, tb
, h
)) {
1087 /* remove the TB from the page list */
1088 if (tb
->page_addr
[0] != page_addr
) {
1089 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1090 tb_page_remove(&p
->first_tb
, tb
);
1091 invalidate_page_bitmap(p
);
1093 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1094 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1095 tb_page_remove(&p
->first_tb
, tb
);
1096 invalidate_page_bitmap(p
);
1099 /* remove the TB from the hash list */
1100 h
= tb_jmp_cache_hash_func(tb
->pc
);
1102 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1103 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1107 /* suppress this TB from the two jump lists */
1108 tb_remove_from_jmp_list(tb
, 0);
1109 tb_remove_from_jmp_list(tb
, 1);
1111 /* suppress any remaining jumps to this TB */
1114 tb_ctx
.tb_phys_invalidate_count
++;
1117 #ifdef CONFIG_SOFTMMU
1118 static void build_page_bitmap(PageDesc
*p
)
1120 int n
, tb_start
, tb_end
;
1121 TranslationBlock
*tb
;
1123 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1126 while (tb
!= NULL
) {
1127 n
= (uintptr_t)tb
& 3;
1128 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1129 /* NOTE: this is subtle as a TB may span two physical pages */
1131 /* NOTE: tb_end may be after the end of the page, but
1132 it is not a problem */
1133 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1134 tb_end
= tb_start
+ tb
->size
;
1135 if (tb_end
> TARGET_PAGE_SIZE
) {
1136 tb_end
= TARGET_PAGE_SIZE
;
1140 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1142 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1143 tb
= tb
->page_next
[n
];
1148 /* add the tb in the target page and protect it if necessary
1150 * Called with mmap_lock held for user-mode emulation.
1152 static inline void tb_alloc_page(TranslationBlock
*tb
,
1153 unsigned int n
, tb_page_addr_t page_addr
)
1156 #ifndef CONFIG_USER_ONLY
1157 bool page_already_protected
;
1160 assert_memory_lock();
1162 tb
->page_addr
[n
] = page_addr
;
1163 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1164 tb
->page_next
[n
] = p
->first_tb
;
1165 #ifndef CONFIG_USER_ONLY
1166 page_already_protected
= p
->first_tb
!= NULL
;
1168 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1169 invalidate_page_bitmap(p
);
1171 #if defined(CONFIG_USER_ONLY)
1172 if (p
->flags
& PAGE_WRITE
) {
1177 /* force the host page as non writable (writes will have a
1178 page fault + mprotect overhead) */
1179 page_addr
&= qemu_host_page_mask
;
1181 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1182 addr
+= TARGET_PAGE_SIZE
) {
1184 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1189 p2
->flags
&= ~PAGE_WRITE
;
1191 mprotect(g2h(page_addr
), qemu_host_page_size
,
1192 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1193 if (DEBUG_TB_INVALIDATE_GATE
) {
1194 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT
"\n", page_addr
);
1198 /* if some code is already present, then the pages are already
1199 protected. So we handle the case where only the first TB is
1200 allocated in a physical page */
1201 if (!page_already_protected
) {
1202 tlb_protect_code(page_addr
);
1207 /* add a new TB and link it to the physical page tables. phys_page2 is
1208 * (-1) to indicate that only one page contains the TB.
1210 * Called with mmap_lock held for user-mode emulation.
1212 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1213 tb_page_addr_t phys_page2
)
1217 assert_memory_lock();
1219 /* add in the page list */
1220 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1221 if (phys_page2
!= -1) {
1222 tb_alloc_page(tb
, 1, phys_page2
);
1224 tb
->page_addr
[1] = -1;
1227 /* add in the hash table */
1228 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->cflags
& CF_HASH_MASK
,
1229 tb
->trace_vcpu_dstate
);
1230 qht_insert(&tb_ctx
.htable
, tb
, h
);
1232 #ifdef CONFIG_USER_ONLY
1233 if (DEBUG_TB_CHECK_GATE
) {
1239 /* Called with mmap_lock held for user mode emulation. */
1240 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1241 target_ulong pc
, target_ulong cs_base
,
1242 uint32_t flags
, int cflags
)
1244 CPUArchState
*env
= cpu
->env_ptr
;
1245 TranslationBlock
*tb
;
1246 tb_page_addr_t phys_pc
, phys_page2
;
1247 target_ulong virt_page2
;
1248 tcg_insn_unit
*gen_code_buf
;
1249 int gen_code_size
, search_size
;
1250 #ifdef CONFIG_PROFILER
1251 TCGProfile
*prof
= &tcg_ctx
->prof
;
1254 assert_memory_lock();
1256 phys_pc
= get_page_addr_code(env
, pc
);
1260 if (unlikely(!tb
)) {
1261 /* flush must be done */
1264 /* Make the execution loop process the flush as soon as possible. */
1265 cpu
->exception_index
= EXCP_INTERRUPT
;
1269 gen_code_buf
= tcg_ctx
->code_gen_ptr
;
1270 tb
->tc
.ptr
= gen_code_buf
;
1272 tb
->cs_base
= cs_base
;
1274 tb
->cflags
= cflags
;
1275 tb
->trace_vcpu_dstate
= *cpu
->trace_dstate
;
1276 tcg_ctx
->tb_cflags
= cflags
;
1278 #ifdef CONFIG_PROFILER
1279 /* includes aborted translations because of exceptions */
1280 atomic_set(&prof
->tb_count1
, prof
->tb_count1
+ 1);
1281 ti
= profile_getclock();
1284 tcg_func_start(tcg_ctx
);
1286 tcg_ctx
->cpu
= ENV_GET_CPU(env
);
1287 gen_intermediate_code(cpu
, tb
);
1288 tcg_ctx
->cpu
= NULL
;
1290 trace_translate_block(tb
, tb
->pc
, tb
->tc
.ptr
);
1292 /* generate machine code */
1293 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1294 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1295 tcg_ctx
->tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1296 if (TCG_TARGET_HAS_direct_jump
) {
1297 tcg_ctx
->tb_jmp_insn_offset
= tb
->jmp_target_arg
;
1298 tcg_ctx
->tb_jmp_target_addr
= NULL
;
1300 tcg_ctx
->tb_jmp_insn_offset
= NULL
;
1301 tcg_ctx
->tb_jmp_target_addr
= tb
->jmp_target_arg
;
1304 #ifdef CONFIG_PROFILER
1305 atomic_set(&prof
->tb_count
, prof
->tb_count
+ 1);
1306 atomic_set(&prof
->interm_time
, prof
->interm_time
+ profile_getclock() - ti
);
1307 ti
= profile_getclock();
1310 /* ??? Overflow could be handled better here. In particular, we
1311 don't need to re-do gen_intermediate_code, nor should we re-do
1312 the tcg optimization currently hidden inside tcg_gen_code. All
1313 that should be required is to flush the TBs, allocate a new TB,
1314 re-initialize it per above, and re-do the actual code generation. */
1315 gen_code_size
= tcg_gen_code(tcg_ctx
, tb
);
1316 if (unlikely(gen_code_size
< 0)) {
1317 goto buffer_overflow
;
1319 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1320 if (unlikely(search_size
< 0)) {
1321 goto buffer_overflow
;
1323 tb
->tc
.size
= gen_code_size
;
1325 #ifdef CONFIG_PROFILER
1326 atomic_set(&prof
->code_time
, prof
->code_time
+ profile_getclock() - ti
);
1327 atomic_set(&prof
->code_in_len
, prof
->code_in_len
+ tb
->size
);
1328 atomic_set(&prof
->code_out_len
, prof
->code_out_len
+ gen_code_size
);
1329 atomic_set(&prof
->search_out_len
, prof
->search_out_len
+ search_size
);
1333 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1334 qemu_log_in_addr_range(tb
->pc
)) {
1336 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1337 if (tcg_ctx
->data_gen_ptr
) {
1338 size_t code_size
= tcg_ctx
->data_gen_ptr
- tb
->tc
.ptr
;
1339 size_t data_size
= gen_code_size
- code_size
;
1342 log_disas(tb
->tc
.ptr
, code_size
);
1344 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1345 if (sizeof(tcg_target_ulong
) == 8) {
1346 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1347 (uintptr_t)tcg_ctx
->data_gen_ptr
+ i
,
1348 *(uint64_t *)(tcg_ctx
->data_gen_ptr
+ i
));
1350 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1351 (uintptr_t)tcg_ctx
->data_gen_ptr
+ i
,
1352 *(uint32_t *)(tcg_ctx
->data_gen_ptr
+ i
));
1356 log_disas(tb
->tc
.ptr
, gen_code_size
);
1364 atomic_set(&tcg_ctx
->code_gen_ptr
, (void *)
1365 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1368 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1369 /* if we are doing vsyscall don't link the page as it lies in high memory
1370 and tb_alloc_page will abort due to page_l1_map returning NULL */
1371 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1372 && phys_pc
< TARGET_VSYSCALL_END
))
1376 /* init jump list */
1377 assert(((uintptr_t)tb
& 3) == 0);
1378 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1379 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1380 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1382 /* init original jump addresses wich has been set during tcg_gen_code() */
1383 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1384 tb_reset_jump(tb
, 0);
1386 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1387 tb_reset_jump(tb
, 1);
1390 /* check next page if needed */
1391 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1393 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1394 phys_page2
= get_page_addr_code(env
, virt_page2
);
1396 /* As long as consistency of the TB stuff is provided by tb_lock in user
1397 * mode and is implicit in single-threaded softmmu emulation, no explicit
1398 * memory barrier is required before tb_link_page() makes the TB visible
1399 * through the physical hash table and physical page list.
1401 tb_link_page(tb
, phys_pc
, phys_page2
);
1402 g_tree_insert(tb_ctx
.tb_tree
, &tb
->tc
, tb
);
1407 * Invalidate all TBs which intersect with the target physical address range
1408 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1409 * 'is_cpu_write_access' should be true if called from a real cpu write
1410 * access: the virtual CPU will exit the current TB if code is modified inside
1413 * Called with mmap_lock held for user-mode emulation, grabs tb_lock
1414 * Called with tb_lock held for system-mode emulation
1416 static void tb_invalidate_phys_range_1(tb_page_addr_t start
, tb_page_addr_t end
)
1418 while (start
< end
) {
1419 tb_invalidate_phys_page_range(start
, end
, 0);
1420 start
&= TARGET_PAGE_MASK
;
1421 start
+= TARGET_PAGE_SIZE
;
1425 #ifdef CONFIG_SOFTMMU
1426 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1429 tb_invalidate_phys_range_1(start
, end
);
1432 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1434 assert_memory_lock();
1436 tb_invalidate_phys_range_1(start
, end
);
1441 * Invalidate all TBs which intersect with the target physical address range
1442 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1443 * 'is_cpu_write_access' should be true if called from a real cpu write
1444 * access: the virtual CPU will exit the current TB if code is modified inside
1447 * Called with tb_lock/mmap_lock held for user-mode emulation
1448 * Called with tb_lock held for system-mode emulation
1450 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1451 int is_cpu_write_access
)
1453 TranslationBlock
*tb
, *tb_next
;
1454 tb_page_addr_t tb_start
, tb_end
;
1457 #ifdef TARGET_HAS_PRECISE_SMC
1458 CPUState
*cpu
= current_cpu
;
1459 CPUArchState
*env
= NULL
;
1460 int current_tb_not_found
= is_cpu_write_access
;
1461 TranslationBlock
*current_tb
= NULL
;
1462 int current_tb_modified
= 0;
1463 target_ulong current_pc
= 0;
1464 target_ulong current_cs_base
= 0;
1465 uint32_t current_flags
= 0;
1466 #endif /* TARGET_HAS_PRECISE_SMC */
1468 assert_memory_lock();
1471 p
= page_find(start
>> TARGET_PAGE_BITS
);
1475 #if defined(TARGET_HAS_PRECISE_SMC)
1481 /* we remove all the TBs in the range [start, end[ */
1482 /* XXX: see if in some cases it could be faster to invalidate all
1485 while (tb
!= NULL
) {
1486 n
= (uintptr_t)tb
& 3;
1487 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1488 tb_next
= tb
->page_next
[n
];
1489 /* NOTE: this is subtle as a TB may span two physical pages */
1491 /* NOTE: tb_end may be after the end of the page, but
1492 it is not a problem */
1493 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1494 tb_end
= tb_start
+ tb
->size
;
1496 tb_start
= tb
->page_addr
[1];
1497 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1499 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1500 #ifdef TARGET_HAS_PRECISE_SMC
1501 if (current_tb_not_found
) {
1502 current_tb_not_found
= 0;
1504 if (cpu
->mem_io_pc
) {
1505 /* now we have a real cpu fault */
1506 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1509 if (current_tb
== tb
&&
1510 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1511 /* If we are modifying the current TB, we must stop
1512 its execution. We could be more precise by checking
1513 that the modification is after the current PC, but it
1514 would require a specialized function to partially
1515 restore the CPU state */
1517 current_tb_modified
= 1;
1518 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1519 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1522 #endif /* TARGET_HAS_PRECISE_SMC */
1523 tb_phys_invalidate(tb
, -1);
1527 #if !defined(CONFIG_USER_ONLY)
1528 /* if no code remaining, no need to continue to use slow writes */
1530 invalidate_page_bitmap(p
);
1531 tlb_unprotect_code(start
);
1534 #ifdef TARGET_HAS_PRECISE_SMC
1535 if (current_tb_modified
) {
1536 /* Force execution of one insn next time. */
1537 cpu
->cflags_next_tb
= 1 | curr_cflags();
1538 cpu_loop_exit_noexc(cpu
);
1543 #ifdef CONFIG_SOFTMMU
1544 /* len must be <= 8 and start must be a multiple of len.
1545 * Called via softmmu_template.h when code areas are written to with
1546 * iothread mutex not held.
1548 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1554 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1555 cpu_single_env
->mem_io_vaddr
, len
,
1556 cpu_single_env
->eip
,
1557 cpu_single_env
->eip
+
1558 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1561 assert_memory_lock();
1563 p
= page_find(start
>> TARGET_PAGE_BITS
);
1567 if (!p
->code_bitmap
&&
1568 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1569 /* build code bitmap. FIXME: writes should be protected by
1570 * tb_lock, reads by tb_lock or RCU.
1572 build_page_bitmap(p
);
1574 if (p
->code_bitmap
) {
1578 nr
= start
& ~TARGET_PAGE_MASK
;
1579 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1580 if (b
& ((1 << len
) - 1)) {
1585 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1589 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1590 * host PC of the faulting store instruction that caused this invalidate.
1591 * Returns true if the caller needs to abort execution of the current
1592 * TB (because it was modified by this store and the guest CPU has
1593 * precise-SMC semantics).
1595 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
1597 TranslationBlock
*tb
;
1600 #ifdef TARGET_HAS_PRECISE_SMC
1601 TranslationBlock
*current_tb
= NULL
;
1602 CPUState
*cpu
= current_cpu
;
1603 CPUArchState
*env
= NULL
;
1604 int current_tb_modified
= 0;
1605 target_ulong current_pc
= 0;
1606 target_ulong current_cs_base
= 0;
1607 uint32_t current_flags
= 0;
1610 assert_memory_lock();
1612 addr
&= TARGET_PAGE_MASK
;
1613 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1620 #ifdef TARGET_HAS_PRECISE_SMC
1621 if (tb
&& pc
!= 0) {
1622 current_tb
= tb_find_pc(pc
);
1628 while (tb
!= NULL
) {
1629 n
= (uintptr_t)tb
& 3;
1630 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1631 #ifdef TARGET_HAS_PRECISE_SMC
1632 if (current_tb
== tb
&&
1633 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1634 /* If we are modifying the current TB, we must stop
1635 its execution. We could be more precise by checking
1636 that the modification is after the current PC, but it
1637 would require a specialized function to partially
1638 restore the CPU state */
1640 current_tb_modified
= 1;
1641 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1642 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1645 #endif /* TARGET_HAS_PRECISE_SMC */
1646 tb_phys_invalidate(tb
, addr
);
1647 tb
= tb
->page_next
[n
];
1650 #ifdef TARGET_HAS_PRECISE_SMC
1651 if (current_tb_modified
) {
1652 /* Force execution of one insn next time. */
1653 cpu
->cflags_next_tb
= 1 | curr_cflags();
1654 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps
1655 * back into the cpu_exec loop. */
1666 * Find the TB 'tb' such that
1667 * tb->tc.ptr <= tc_ptr < tb->tc.ptr + tb->tc.size
1668 * Return NULL if not found.
1670 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1672 struct tb_tc s
= { .ptr
= (void *)tc_ptr
};
1674 return g_tree_lookup(tb_ctx
.tb_tree
, &s
);
1677 #if !defined(CONFIG_USER_ONLY)
1678 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1680 ram_addr_t ram_addr
;
1685 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1686 if (!(memory_region_is_ram(mr
)
1687 || memory_region_is_romd(mr
))) {
1691 ram_addr
= memory_region_get_ram_addr(mr
) + addr
;
1693 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1697 #endif /* !defined(CONFIG_USER_ONLY) */
1699 /* Called with tb_lock held. */
1700 void tb_check_watchpoint(CPUState
*cpu
)
1702 TranslationBlock
*tb
;
1704 tb
= tb_find_pc(cpu
->mem_io_pc
);
1706 /* We can use retranslation to find the PC. */
1707 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1708 tb_phys_invalidate(tb
, -1);
1710 /* The exception probably happened in a helper. The CPU state should
1711 have been saved before calling it. Fetch the PC from there. */
1712 CPUArchState
*env
= cpu
->env_ptr
;
1713 target_ulong pc
, cs_base
;
1714 tb_page_addr_t addr
;
1717 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1718 addr
= get_page_addr_code(env
, pc
);
1719 tb_invalidate_phys_range(addr
, addr
+ 1);
1723 #ifndef CONFIG_USER_ONLY
1724 /* in deterministic execution mode, instructions doing device I/Os
1725 * must be at the end of the TB.
1727 * Called by softmmu_template.h, with iothread mutex not held.
1729 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1731 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1732 CPUArchState
*env
= cpu
->env_ptr
;
1734 TranslationBlock
*tb
;
1738 tb
= tb_find_pc(retaddr
);
1740 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1743 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1744 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1745 /* Calculate how many instructions had been executed before the fault
1747 n
= n
- cpu
->icount_decr
.u16
.low
;
1748 /* Generate a new TB ending on the I/O insn. */
1750 /* On MIPS and SH, delay slot instructions can only be restarted if
1751 they were already the first instruction in the TB. If this is not
1752 the first instruction in a TB then re-execute the preceding
1754 #if defined(TARGET_MIPS)
1755 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1756 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1757 cpu
->icount_decr
.u16
.low
++;
1758 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1760 #elif defined(TARGET_SH4)
1761 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1764 cpu
->icount_decr
.u16
.low
++;
1765 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1768 /* This should never happen. */
1769 if (n
> CF_COUNT_MASK
) {
1770 cpu_abort(cpu
, "TB too big during recompile");
1773 /* Adjust the execution state of the next TB. */
1774 cpu
->cflags_next_tb
= curr_cflags() | CF_LAST_IO
| n
;
1776 if (tb
->cflags
& CF_NOCACHE
) {
1778 /* Invalidate original TB if this TB was generated in
1779 * cpu_exec_nocache() */
1780 tb_phys_invalidate(tb
->orig_tb
, -1);
1785 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1786 * the first in the TB) then we end up generating a whole new TB and
1787 * repeating the fault, which is horribly inefficient.
1788 * Better would be to execute just this insn uncached, or generate a
1791 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the
1792 * tb_lock gets reset.
1794 cpu_loop_exit_noexc(cpu
);
1797 static void tb_jmp_cache_clear_page(CPUState
*cpu
, target_ulong page_addr
)
1799 unsigned int i
, i0
= tb_jmp_cache_hash_page(page_addr
);
1801 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
1802 atomic_set(&cpu
->tb_jmp_cache
[i0
+ i
], NULL
);
1806 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1808 /* Discard jump cache entries for any tb which might potentially
1809 overlap the flushed page. */
1810 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
1811 tb_jmp_cache_clear_page(cpu
, addr
);
1814 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
1815 struct qht_stats hst
)
1817 uint32_t hgram_opts
;
1821 if (!hst
.head_buckets
) {
1824 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1825 hst
.used_head_buckets
, hst
.head_buckets
,
1826 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
1828 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1829 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
1830 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
1831 hgram_opts
|= QDIST_PR_NODECIMAL
;
1833 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
1834 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1835 qdist_avg(&hst
.occupancy
) * 100, hgram
);
1838 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1839 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
1840 if (hgram_bins
> 10) {
1844 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
1846 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
1847 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1848 qdist_avg(&hst
.chain
), hgram
);
1852 struct tb_tree_stats
{
1855 size_t max_target_size
;
1856 size_t direct_jmp_count
;
1857 size_t direct_jmp2_count
;
1861 static gboolean
tb_tree_stats_iter(gpointer key
, gpointer value
, gpointer data
)
1863 const TranslationBlock
*tb
= value
;
1864 struct tb_tree_stats
*tst
= data
;
1866 tst
->host_size
+= tb
->tc
.size
;
1867 tst
->target_size
+= tb
->size
;
1868 if (tb
->size
> tst
->max_target_size
) {
1869 tst
->max_target_size
= tb
->size
;
1871 if (tb
->page_addr
[1] != -1) {
1874 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1875 tst
->direct_jmp_count
++;
1876 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1877 tst
->direct_jmp2_count
++;
1883 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1885 struct tb_tree_stats tst
= {};
1886 struct qht_stats hst
;
1891 nb_tbs
= g_tree_nnodes(tb_ctx
.tb_tree
);
1892 g_tree_foreach(tb_ctx
.tb_tree
, tb_tree_stats_iter
, &tst
);
1893 /* XXX: avoid using doubles ? */
1894 cpu_fprintf(f
, "Translation buffer state:\n");
1896 * Report total code size including the padding and TB structs;
1897 * otherwise users might think "-tb-size" is not honoured.
1898 * For avg host size we use the precise numbers from tb_tree_stats though.
1900 cpu_fprintf(f
, "gen code size %zu/%zu\n",
1901 tcg_code_size(), tcg_code_capacity());
1902 cpu_fprintf(f
, "TB count %zu\n", nb_tbs
);
1903 cpu_fprintf(f
, "TB avg target size %zu max=%zu bytes\n",
1904 nb_tbs
? tst
.target_size
/ nb_tbs
: 0,
1905 tst
.max_target_size
);
1906 cpu_fprintf(f
, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n",
1907 nb_tbs
? tst
.host_size
/ nb_tbs
: 0,
1908 tst
.target_size
? (double)tst
.host_size
/ tst
.target_size
: 0);
1909 cpu_fprintf(f
, "cross page TB count %zu (%zu%%)\n", tst
.cross_page
,
1910 nb_tbs
? (tst
.cross_page
* 100) / nb_tbs
: 0);
1911 cpu_fprintf(f
, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n",
1912 tst
.direct_jmp_count
,
1913 nb_tbs
? (tst
.direct_jmp_count
* 100) / nb_tbs
: 0,
1914 tst
.direct_jmp2_count
,
1915 nb_tbs
? (tst
.direct_jmp2_count
* 100) / nb_tbs
: 0);
1917 qht_statistics_init(&tb_ctx
.htable
, &hst
);
1918 print_qht_statistics(f
, cpu_fprintf
, hst
);
1919 qht_statistics_destroy(&hst
);
1921 cpu_fprintf(f
, "\nStatistics:\n");
1922 cpu_fprintf(f
, "TB flush count %u\n",
1923 atomic_read(&tb_ctx
.tb_flush_count
));
1924 cpu_fprintf(f
, "TB invalidate count %d\n", tb_ctx
.tb_phys_invalidate_count
);
1925 cpu_fprintf(f
, "TLB flush count %zu\n", tlb_flush_count());
1926 tcg_dump_info(f
, cpu_fprintf
);
1931 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1933 tcg_dump_op_count(f
, cpu_fprintf
);
1936 #else /* CONFIG_USER_ONLY */
1938 void cpu_interrupt(CPUState
*cpu
, int mask
)
1940 g_assert(qemu_mutex_iothread_locked());
1941 cpu
->interrupt_request
|= mask
;
1942 cpu
->icount_decr
.u16
.high
= -1;
1946 * Walks guest process memory "regions" one by one
1947 * and calls callback function 'fn' for each region.
1949 struct walk_memory_regions_data
{
1950 walk_memory_regions_fn fn
;
1956 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1957 target_ulong end
, int new_prot
)
1959 if (data
->start
!= -1u) {
1960 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1966 data
->start
= (new_prot
? end
: -1u);
1967 data
->prot
= new_prot
;
1972 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1973 target_ulong base
, int level
, void **lp
)
1979 return walk_memory_regions_end(data
, base
, 0);
1985 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1986 int prot
= pd
[i
].flags
;
1988 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1989 if (prot
!= data
->prot
) {
1990 rc
= walk_memory_regions_end(data
, pa
, prot
);
1999 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2000 pa
= base
| ((target_ulong
)i
<<
2001 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
2002 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2012 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2014 struct walk_memory_regions_data data
;
2015 uintptr_t i
, l1_sz
= v_l1_size
;
2022 for (i
= 0; i
< l1_sz
; i
++) {
2023 target_ulong base
= i
<< (v_l1_shift
+ TARGET_PAGE_BITS
);
2024 int rc
= walk_memory_regions_1(&data
, base
, v_l2_levels
, l1_map
+ i
);
2030 return walk_memory_regions_end(&data
, 0, 0);
2033 static int dump_region(void *priv
, target_ulong start
,
2034 target_ulong end
, abi_ulong prot
)
2036 FILE *f
= (FILE *)priv
;
2038 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
2039 " "TARGET_FMT_lx
" %c%c%c\n",
2040 start
, end
, end
- start
,
2041 ((prot
& PAGE_READ
) ? 'r' : '-'),
2042 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2043 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2048 /* dump memory mappings */
2049 void page_dump(FILE *f
)
2051 const int length
= sizeof(target_ulong
) * 2;
2052 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
2053 length
, "start", length
, "end", length
, "size", "prot");
2054 walk_memory_regions(f
, dump_region
);
2057 int page_get_flags(target_ulong address
)
2061 p
= page_find(address
>> TARGET_PAGE_BITS
);
2068 /* Modify the flags of a page and invalidate the code if necessary.
2069 The flag PAGE_WRITE_ORG is positioned automatically depending
2070 on PAGE_WRITE. The mmap_lock should already be held. */
2071 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2073 target_ulong addr
, len
;
2075 /* This function should never be called with addresses outside the
2076 guest address space. If this assert fires, it probably indicates
2077 a missing call to h2g_valid. */
2078 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2079 assert(end
<= ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2081 assert(start
< end
);
2082 assert_memory_lock();
2084 start
= start
& TARGET_PAGE_MASK
;
2085 end
= TARGET_PAGE_ALIGN(end
);
2087 if (flags
& PAGE_WRITE
) {
2088 flags
|= PAGE_WRITE_ORG
;
2091 for (addr
= start
, len
= end
- start
;
2093 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2094 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2096 /* If the write protection bit is set, then we invalidate
2098 if (!(p
->flags
& PAGE_WRITE
) &&
2099 (flags
& PAGE_WRITE
) &&
2101 tb_invalidate_phys_page(addr
, 0);
2107 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2113 /* This function should never be called with addresses outside the
2114 guest address space. If this assert fires, it probably indicates
2115 a missing call to h2g_valid. */
2116 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2117 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2123 if (start
+ len
- 1 < start
) {
2124 /* We've wrapped around. */
2128 /* must do before we loose bits in the next step */
2129 end
= TARGET_PAGE_ALIGN(start
+ len
);
2130 start
= start
& TARGET_PAGE_MASK
;
2132 for (addr
= start
, len
= end
- start
;
2134 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2135 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2139 if (!(p
->flags
& PAGE_VALID
)) {
2143 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
2146 if (flags
& PAGE_WRITE
) {
2147 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2150 /* unprotect the page if it was put read-only because it
2151 contains translated code */
2152 if (!(p
->flags
& PAGE_WRITE
)) {
2153 if (!page_unprotect(addr
, 0)) {
2162 /* called from signal handler: invalidate the code and unprotect the
2163 * page. Return 0 if the fault was not handled, 1 if it was handled,
2164 * and 2 if it was handled but the caller must cause the TB to be
2165 * immediately exited. (We can only return 2 if the 'pc' argument is
2168 int page_unprotect(target_ulong address
, uintptr_t pc
)
2171 bool current_tb_invalidated
;
2173 target_ulong host_start
, host_end
, addr
;
2175 /* Technically this isn't safe inside a signal handler. However we
2176 know this only ever happens in a synchronous SEGV handler, so in
2177 practice it seems to be ok. */
2180 p
= page_find(address
>> TARGET_PAGE_BITS
);
2186 /* if the page was really writable, then we change its
2187 protection back to writable */
2188 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2189 host_start
= address
& qemu_host_page_mask
;
2190 host_end
= host_start
+ qemu_host_page_size
;
2193 current_tb_invalidated
= false;
2194 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2195 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2196 p
->flags
|= PAGE_WRITE
;
2199 /* and since the content will be modified, we must invalidate
2200 the corresponding translated code. */
2201 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2202 #ifdef CONFIG_USER_ONLY
2203 if (DEBUG_TB_CHECK_GATE
) {
2204 tb_invalidate_check(addr
);
2208 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2212 /* If current TB was invalidated return to main loop */
2213 return current_tb_invalidated
? 2 : 1;
2218 #endif /* CONFIG_USER_ONLY */
2220 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2221 void tcg_flush_softmmu_tlb(CPUState
*cs
)
2223 #ifdef CONFIG_SOFTMMU