4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
21 #include "qemu-common.h"
22 #define NO_CPU_IO_DEFS
25 #include "disas/disas.h"
26 #include "exec/exec-all.h"
28 #if defined(CONFIG_USER_ONLY)
30 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
31 #include <sys/param.h>
32 #if __FreeBSD_version >= 700104
33 #define HAVE_KINFO_GETVMMAP
34 #define sigqueue sigqueue_freebsd /* avoid redefinition */
36 #include <machine/profile.h>
45 #include "exec/ram_addr.h"
48 #include "exec/cputlb.h"
49 #include "exec/tb-hash.h"
50 #include "translate-all.h"
51 #include "qemu/bitmap.h"
52 #include "qemu/error-report.h"
53 #include "qemu/timer.h"
54 #include "qemu/main-loop.h"
56 #include "sysemu/cpus.h"
58 /* #define DEBUG_TB_INVALIDATE */
59 /* #define DEBUG_TB_FLUSH */
60 /* make various TB consistency checks */
61 /* #define DEBUG_TB_CHECK */
63 #ifdef DEBUG_TB_INVALIDATE
64 #define DEBUG_TB_INVALIDATE_GATE 1
66 #define DEBUG_TB_INVALIDATE_GATE 0
70 #define DEBUG_TB_FLUSH_GATE 1
72 #define DEBUG_TB_FLUSH_GATE 0
75 #if !defined(CONFIG_USER_ONLY)
76 /* TB consistency checks only implemented for usermode emulation. */
81 #define DEBUG_TB_CHECK_GATE 1
83 #define DEBUG_TB_CHECK_GATE 0
86 /* Access to the various translations structures need to be serialised via locks
88 * In user-mode emulation access to the memory related structures are protected
90 * In !user-mode we use per-page locks.
93 #define assert_memory_lock()
95 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
98 #define SMC_BITMAP_USE_THRESHOLD 10
100 typedef struct PageDesc
{
101 /* list of TBs intersecting this ram page */
103 #ifdef CONFIG_SOFTMMU
104 /* in order to optimize self modifying code, we count the number
105 of lookups we do to a given page to use a bitmap */
106 unsigned long *code_bitmap
;
107 unsigned int code_write_count
;
111 #ifndef CONFIG_USER_ONLY
117 * struct page_entry - page descriptor entry
118 * @pd: pointer to the &struct PageDesc of the page this entry represents
119 * @index: page index of the page
120 * @locked: whether the page is locked
122 * This struct helps us keep track of the locked state of a page, without
123 * bloating &struct PageDesc.
125 * A page lock protects accesses to all fields of &struct PageDesc.
127 * See also: &struct page_collection.
131 tb_page_addr_t index
;
136 * struct page_collection - tracks a set of pages (i.e. &struct page_entry's)
137 * @tree: Binary search tree (BST) of the pages, with key == page index
138 * @max: Pointer to the page in @tree with the highest page index
140 * To avoid deadlock we lock pages in ascending order of page index.
141 * When operating on a set of pages, we need to keep track of them so that
142 * we can lock them in order and also unlock them later. For this we collect
143 * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the
144 * @tree implementation we use does not provide an O(1) operation to obtain the
145 * highest-ranked element, we use @max to keep track of the inserted page
146 * with the highest index. This is valuable because if a page is not in
147 * the tree and its index is higher than @max's, then we can lock it
148 * without breaking the locking order rule.
150 * Note on naming: 'struct page_set' would be shorter, but we already have a few
151 * page_set_*() helpers, so page_collection is used instead to avoid confusion.
153 * See also: page_collection_lock().
155 struct page_collection
{
157 struct page_entry
*max
;
160 /* list iterators for lists of tagged pointers in TranslationBlock */
161 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \
162 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
163 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
164 tb = (TranslationBlock *)((uintptr_t)tb & ~1))
166 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
167 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
169 #define TB_FOR_EACH_JMP(head_tb, tb, n) \
170 TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
172 /* In system mode we want L1_MAP to be based on ram offsets,
173 while in user mode we want it to be based on virtual addresses. */
174 #if !defined(CONFIG_USER_ONLY)
175 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
176 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
178 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
181 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
184 /* Size of the L2 (and L3, etc) page tables. */
186 #define V_L2_SIZE (1 << V_L2_BITS)
188 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
189 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS
>
190 sizeof_field(TranslationBlock
, trace_vcpu_dstate
)
194 * L1 Mapping properties
196 static int v_l1_size
;
197 static int v_l1_shift
;
198 static int v_l2_levels
;
200 /* The bottom level has pointers to PageDesc, and is indexed by
201 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
203 #define V_L1_MIN_BITS 4
204 #define V_L1_MAX_BITS (V_L2_BITS + 3)
205 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
207 static void *l1_map
[V_L1_MAX_SIZE
];
209 /* code generation context */
210 TCGContext tcg_init_ctx
;
211 __thread TCGContext
*tcg_ctx
;
215 static void page_table_config_init(void)
219 assert(TARGET_PAGE_BITS
);
220 /* The bits remaining after N lower levels of page tables. */
221 v_l1_bits
= (L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
) % V_L2_BITS
;
222 if (v_l1_bits
< V_L1_MIN_BITS
) {
223 v_l1_bits
+= V_L2_BITS
;
226 v_l1_size
= 1 << v_l1_bits
;
227 v_l1_shift
= L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
- v_l1_bits
;
228 v_l2_levels
= v_l1_shift
/ V_L2_BITS
- 1;
230 assert(v_l1_bits
<= V_L1_MAX_BITS
);
231 assert(v_l1_shift
% V_L2_BITS
== 0);
232 assert(v_l2_levels
>= 0);
235 void cpu_gen_init(void)
237 tcg_context_init(&tcg_init_ctx
);
240 /* Encode VAL as a signed leb128 sequence at P.
241 Return P incremented past the encoded value. */
242 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
249 more
= !((val
== 0 && (byte
& 0x40) == 0)
250 || (val
== -1 && (byte
& 0x40) != 0));
260 /* Decode a signed leb128 sequence at *PP; increment *PP past the
261 decoded value. Return the decoded value. */
262 static target_long
decode_sleb128(uint8_t **pp
)
270 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
272 } while (byte
& 0x80);
273 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
274 val
|= -(target_ulong
)1 << shift
;
281 /* Encode the data collected about the instructions while compiling TB.
282 Place the data at BLOCK, and return the number of bytes consumed.
284 The logical table consists of TARGET_INSN_START_WORDS target_ulong's,
285 which come from the target's insn_start data, followed by a uintptr_t
286 which comes from the host pc of the end of the code implementing the insn.
288 Each line of the table is encoded as sleb128 deltas from the previous
289 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
290 That is, the first column is seeded with the guest pc, the last column
291 with the host pc, and the middle columns with zeros. */
293 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
295 uint8_t *highwater
= tcg_ctx
->code_gen_highwater
;
299 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
302 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
304 prev
= (j
== 0 ? tb
->pc
: 0);
306 prev
= tcg_ctx
->gen_insn_data
[i
- 1][j
];
308 p
= encode_sleb128(p
, tcg_ctx
->gen_insn_data
[i
][j
] - prev
);
310 prev
= (i
== 0 ? 0 : tcg_ctx
->gen_insn_end_off
[i
- 1]);
311 p
= encode_sleb128(p
, tcg_ctx
->gen_insn_end_off
[i
] - prev
);
313 /* Test for (pending) buffer overflow. The assumption is that any
314 one row beginning below the high water mark cannot overrun
315 the buffer completely. Thus we can test for overflow after
316 encoding a row without having to check during encoding. */
317 if (unlikely(p
> highwater
)) {
325 /* The cpu state corresponding to 'searched_pc' is restored.
326 * When reset_icount is true, current TB will be interrupted and
327 * icount should be recalculated.
329 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
330 uintptr_t searched_pc
, bool reset_icount
)
332 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
333 uintptr_t host_pc
= (uintptr_t)tb
->tc
.ptr
;
334 CPUArchState
*env
= cpu
->env_ptr
;
335 uint8_t *p
= tb
->tc
.ptr
+ tb
->tc
.size
;
336 int i
, j
, num_insns
= tb
->icount
;
337 #ifdef CONFIG_PROFILER
338 TCGProfile
*prof
= &tcg_ctx
->prof
;
339 int64_t ti
= profile_getclock();
342 searched_pc
-= GETPC_ADJ
;
344 if (searched_pc
< host_pc
) {
348 /* Reconstruct the stored insn data while looking for the point at
349 which the end of the insn exceeds the searched_pc. */
350 for (i
= 0; i
< num_insns
; ++i
) {
351 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
352 data
[j
] += decode_sleb128(&p
);
354 host_pc
+= decode_sleb128(&p
);
355 if (host_pc
> searched_pc
) {
362 if (reset_icount
&& (tb_cflags(tb
) & CF_USE_ICOUNT
)) {
364 /* Reset the cycle counter to the start of the block
365 and shift if to the number of actually executed instructions */
366 cpu
->icount_decr
.u16
.low
+= num_insns
- i
;
368 restore_state_to_opc(env
, tb
, data
);
370 #ifdef CONFIG_PROFILER
371 atomic_set(&prof
->restore_time
,
372 prof
->restore_time
+ profile_getclock() - ti
);
373 atomic_set(&prof
->restore_count
, prof
->restore_count
+ 1);
378 bool cpu_restore_state(CPUState
*cpu
, uintptr_t host_pc
, bool will_exit
)
380 TranslationBlock
*tb
;
382 uintptr_t check_offset
;
384 /* The host_pc has to be in the region of current code buffer. If
385 * it is not we will not be able to resolve it here. The two cases
386 * where host_pc will not be correct are:
388 * - fault during translation (instruction fetch)
389 * - fault from helper (not using GETPC() macro)
391 * Either way we need return early as we can't resolve it here.
393 * We are using unsigned arithmetic so if host_pc <
394 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way
395 * above the code_gen_buffer_size
397 check_offset
= host_pc
- (uintptr_t) tcg_init_ctx
.code_gen_buffer
;
399 if (check_offset
< tcg_init_ctx
.code_gen_buffer_size
) {
400 tb
= tcg_tb_lookup(host_pc
);
402 cpu_restore_state_from_tb(cpu
, tb
, host_pc
, will_exit
);
403 if (tb_cflags(tb
) & CF_NOCACHE
) {
404 /* one-shot translation, invalidate it immediately */
405 tb_phys_invalidate(tb
, -1);
415 static void page_init(void)
418 page_table_config_init();
420 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
422 #ifdef HAVE_KINFO_GETVMMAP
423 struct kinfo_vmentry
*freep
;
426 freep
= kinfo_getvmmap(getpid(), &cnt
);
429 for (i
= 0; i
< cnt
; i
++) {
430 unsigned long startaddr
, endaddr
;
432 startaddr
= freep
[i
].kve_start
;
433 endaddr
= freep
[i
].kve_end
;
434 if (h2g_valid(startaddr
)) {
435 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
437 if (h2g_valid(endaddr
)) {
438 endaddr
= h2g(endaddr
);
439 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
441 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
443 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
454 last_brk
= (unsigned long)sbrk(0);
456 f
= fopen("/compat/linux/proc/self/maps", "r");
461 unsigned long startaddr
, endaddr
;
464 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
466 if (n
== 2 && h2g_valid(startaddr
)) {
467 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
469 if (h2g_valid(endaddr
)) {
470 endaddr
= h2g(endaddr
);
474 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
486 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
492 /* Level 1. Always allocated. */
493 lp
= l1_map
+ ((index
>> v_l1_shift
) & (v_l1_size
- 1));
496 for (i
= v_l2_levels
; i
> 0; i
--) {
497 void **p
= atomic_rcu_read(lp
);
505 p
= g_new0(void *, V_L2_SIZE
);
506 existing
= atomic_cmpxchg(lp
, NULL
, p
);
507 if (unlikely(existing
)) {
513 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
516 pd
= atomic_rcu_read(lp
);
523 pd
= g_new0(PageDesc
, V_L2_SIZE
);
524 #ifndef CONFIG_USER_ONLY
528 for (i
= 0; i
< V_L2_SIZE
; i
++) {
529 qemu_spin_init(&pd
[i
].lock
);
533 existing
= atomic_cmpxchg(lp
, NULL
, pd
);
534 if (unlikely(existing
)) {
540 return pd
+ (index
& (V_L2_SIZE
- 1));
543 static inline PageDesc
*page_find(tb_page_addr_t index
)
545 return page_find_alloc(index
, 0);
548 static void page_lock_pair(PageDesc
**ret_p1
, tb_page_addr_t phys1
,
549 PageDesc
**ret_p2
, tb_page_addr_t phys2
, int alloc
);
551 /* In user-mode page locks aren't used; mmap_lock is enough */
552 #ifdef CONFIG_USER_ONLY
554 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
556 static inline void page_lock(PageDesc
*pd
)
559 static inline void page_unlock(PageDesc
*pd
)
562 static inline void page_lock_tb(const TranslationBlock
*tb
)
565 static inline void page_unlock_tb(const TranslationBlock
*tb
)
568 struct page_collection
*
569 page_collection_lock(tb_page_addr_t start
, tb_page_addr_t end
)
574 void page_collection_unlock(struct page_collection
*set
)
576 #else /* !CONFIG_USER_ONLY */
578 #ifdef CONFIG_DEBUG_TCG
580 static __thread GHashTable
*ht_pages_locked_debug
;
582 static void ht_pages_locked_debug_init(void)
584 if (ht_pages_locked_debug
) {
587 ht_pages_locked_debug
= g_hash_table_new(NULL
, NULL
);
590 static bool page_is_locked(const PageDesc
*pd
)
594 ht_pages_locked_debug_init();
595 found
= g_hash_table_lookup(ht_pages_locked_debug
, pd
);
599 static void page_lock__debug(PageDesc
*pd
)
601 ht_pages_locked_debug_init();
602 g_assert(!page_is_locked(pd
));
603 g_hash_table_insert(ht_pages_locked_debug
, pd
, pd
);
606 static void page_unlock__debug(const PageDesc
*pd
)
610 ht_pages_locked_debug_init();
611 g_assert(page_is_locked(pd
));
612 removed
= g_hash_table_remove(ht_pages_locked_debug
, pd
);
617 do_assert_page_locked(const PageDesc
*pd
, const char *file
, int line
)
619 if (unlikely(!page_is_locked(pd
))) {
620 error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
626 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
628 void assert_no_pages_locked(void)
630 ht_pages_locked_debug_init();
631 g_assert(g_hash_table_size(ht_pages_locked_debug
) == 0);
634 #else /* !CONFIG_DEBUG_TCG */
636 #define assert_page_locked(pd)
638 static inline void page_lock__debug(const PageDesc
*pd
)
642 static inline void page_unlock__debug(const PageDesc
*pd
)
646 #endif /* CONFIG_DEBUG_TCG */
648 static inline void page_lock(PageDesc
*pd
)
650 page_lock__debug(pd
);
651 qemu_spin_lock(&pd
->lock
);
654 static inline void page_unlock(PageDesc
*pd
)
656 qemu_spin_unlock(&pd
->lock
);
657 page_unlock__debug(pd
);
660 /* lock the page(s) of a TB in the correct acquisition order */
661 static inline void page_lock_tb(const TranslationBlock
*tb
)
663 page_lock_pair(NULL
, tb
->page_addr
[0], NULL
, tb
->page_addr
[1], 0);
666 static inline void page_unlock_tb(const TranslationBlock
*tb
)
668 PageDesc
*p1
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
671 if (unlikely(tb
->page_addr
[1] != -1)) {
672 PageDesc
*p2
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
680 static inline struct page_entry
*
681 page_entry_new(PageDesc
*pd
, tb_page_addr_t index
)
683 struct page_entry
*pe
= g_malloc(sizeof(*pe
));
691 static void page_entry_destroy(gpointer p
)
693 struct page_entry
*pe
= p
;
695 g_assert(pe
->locked
);
700 /* returns false on success */
701 static bool page_entry_trylock(struct page_entry
*pe
)
705 busy
= qemu_spin_trylock(&pe
->pd
->lock
);
707 g_assert(!pe
->locked
);
709 page_lock__debug(pe
->pd
);
714 static void do_page_entry_lock(struct page_entry
*pe
)
717 g_assert(!pe
->locked
);
721 static gboolean
page_entry_lock(gpointer key
, gpointer value
, gpointer data
)
723 struct page_entry
*pe
= value
;
725 do_page_entry_lock(pe
);
729 static gboolean
page_entry_unlock(gpointer key
, gpointer value
, gpointer data
)
731 struct page_entry
*pe
= value
;
741 * Trylock a page, and if successful, add the page to a collection.
742 * Returns true ("busy") if the page could not be locked; false otherwise.
744 static bool page_trylock_add(struct page_collection
*set
, tb_page_addr_t addr
)
746 tb_page_addr_t index
= addr
>> TARGET_PAGE_BITS
;
747 struct page_entry
*pe
;
750 pe
= g_tree_lookup(set
->tree
, &index
);
755 pd
= page_find(index
);
760 pe
= page_entry_new(pd
, index
);
761 g_tree_insert(set
->tree
, &pe
->index
, pe
);
764 * If this is either (1) the first insertion or (2) a page whose index
765 * is higher than any other so far, just lock the page and move on.
767 if (set
->max
== NULL
|| pe
->index
> set
->max
->index
) {
769 do_page_entry_lock(pe
);
773 * Try to acquire out-of-order lock; if busy, return busy so that we acquire
776 return page_entry_trylock(pe
);
779 static gint
tb_page_addr_cmp(gconstpointer ap
, gconstpointer bp
, gpointer udata
)
781 tb_page_addr_t a
= *(const tb_page_addr_t
*)ap
;
782 tb_page_addr_t b
= *(const tb_page_addr_t
*)bp
;
793 * Lock a range of pages ([@start,@end[) as well as the pages of all
795 * Locking order: acquire locks in ascending order of page index.
797 struct page_collection
*
798 page_collection_lock(tb_page_addr_t start
, tb_page_addr_t end
)
800 struct page_collection
*set
= g_malloc(sizeof(*set
));
801 tb_page_addr_t index
;
804 start
>>= TARGET_PAGE_BITS
;
805 end
>>= TARGET_PAGE_BITS
;
806 g_assert(start
<= end
);
808 set
->tree
= g_tree_new_full(tb_page_addr_cmp
, NULL
, NULL
,
811 assert_no_pages_locked();
814 g_tree_foreach(set
->tree
, page_entry_lock
, NULL
);
816 for (index
= start
; index
<= end
; index
++) {
817 TranslationBlock
*tb
;
820 pd
= page_find(index
);
824 if (page_trylock_add(set
, index
<< TARGET_PAGE_BITS
)) {
825 g_tree_foreach(set
->tree
, page_entry_unlock
, NULL
);
828 assert_page_locked(pd
);
829 PAGE_FOR_EACH_TB(pd
, tb
, n
) {
830 if (page_trylock_add(set
, tb
->page_addr
[0]) ||
831 (tb
->page_addr
[1] != -1 &&
832 page_trylock_add(set
, tb
->page_addr
[1]))) {
833 /* drop all locks, and reacquire in order */
834 g_tree_foreach(set
->tree
, page_entry_unlock
, NULL
);
842 void page_collection_unlock(struct page_collection
*set
)
844 /* entries are unlocked and freed via page_entry_destroy */
845 g_tree_destroy(set
->tree
);
849 #endif /* !CONFIG_USER_ONLY */
851 static void page_lock_pair(PageDesc
**ret_p1
, tb_page_addr_t phys1
,
852 PageDesc
**ret_p2
, tb_page_addr_t phys2
, int alloc
)
855 tb_page_addr_t page1
;
856 tb_page_addr_t page2
;
858 assert_memory_lock();
859 g_assert(phys1
!= -1);
861 page1
= phys1
>> TARGET_PAGE_BITS
;
862 page2
= phys2
>> TARGET_PAGE_BITS
;
864 p1
= page_find_alloc(page1
, alloc
);
868 if (likely(phys2
== -1)) {
871 } else if (page1
== page2
) {
878 p2
= page_find_alloc(page2
, alloc
);
891 #if defined(CONFIG_USER_ONLY)
892 /* Currently it is not recommended to allocate big chunks of data in
893 user mode. It will change when a dedicated libc will be used. */
894 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
895 region in which the guest needs to run. Revisit this. */
896 #define USE_STATIC_CODE_GEN_BUFFER
899 /* Minimum size of the code gen buffer. This number is randomly chosen,
900 but not so small that we can't have a fair number of TB's live. */
901 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
903 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
904 indicated, this is constrained by the range of direct branches on the
905 host cpu, as used by the TCG implementation of goto_tb. */
906 #if defined(__x86_64__)
907 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
908 #elif defined(__sparc__)
909 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
910 #elif defined(__powerpc64__)
911 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
912 #elif defined(__powerpc__)
913 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
914 #elif defined(__aarch64__)
915 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
916 #elif defined(__s390x__)
917 /* We have a +- 4GB range on the branches; leave some slop. */
918 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
919 #elif defined(__mips__)
920 /* We have a 256MB branch region, but leave room to make sure the
921 main executable is also within that region. */
922 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
924 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
927 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
929 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
930 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
931 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
933 static inline size_t size_code_gen_buffer(size_t tb_size
)
935 /* Size the buffer. */
937 #ifdef USE_STATIC_CODE_GEN_BUFFER
938 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
940 /* ??? Needs adjustments. */
941 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
942 static buffer, we could size this on RESERVED_VA, on the text
943 segment size of the executable, or continue to use the default. */
944 tb_size
= (unsigned long)(ram_size
/ 4);
947 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
948 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
950 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
951 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
957 /* In order to use J and JAL within the code_gen_buffer, we require
958 that the buffer not cross a 256MB boundary. */
959 static inline bool cross_256mb(void *addr
, size_t size
)
961 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
964 /* We weren't able to allocate a buffer without crossing that boundary,
965 so make do with the larger portion of the buffer that doesn't cross.
966 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
967 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
969 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
970 size_t size2
= buf1
+ size1
- buf2
;
978 tcg_ctx
->code_gen_buffer_size
= size1
;
983 #ifdef USE_STATIC_CODE_GEN_BUFFER
984 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
985 __attribute__((aligned(CODE_GEN_ALIGN
)));
987 static inline void *alloc_code_gen_buffer(void)
989 void *buf
= static_code_gen_buffer
;
990 void *end
= static_code_gen_buffer
+ sizeof(static_code_gen_buffer
);
993 /* page-align the beginning and end of the buffer */
994 buf
= QEMU_ALIGN_PTR_UP(buf
, qemu_real_host_page_size
);
995 end
= QEMU_ALIGN_PTR_DOWN(end
, qemu_real_host_page_size
);
999 /* Honor a command-line option limiting the size of the buffer. */
1000 if (size
> tcg_ctx
->code_gen_buffer_size
) {
1001 size
= QEMU_ALIGN_DOWN(tcg_ctx
->code_gen_buffer_size
,
1002 qemu_real_host_page_size
);
1004 tcg_ctx
->code_gen_buffer_size
= size
;
1007 if (cross_256mb(buf
, size
)) {
1008 buf
= split_cross_256mb(buf
, size
);
1009 size
= tcg_ctx
->code_gen_buffer_size
;
1013 if (qemu_mprotect_rwx(buf
, size
)) {
1016 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
1020 #elif defined(_WIN32)
1021 static inline void *alloc_code_gen_buffer(void)
1023 size_t size
= tcg_ctx
->code_gen_buffer_size
;
1024 return VirtualAlloc(NULL
, size
, MEM_RESERVE
| MEM_COMMIT
,
1025 PAGE_EXECUTE_READWRITE
);
1028 static inline void *alloc_code_gen_buffer(void)
1030 int prot
= PROT_WRITE
| PROT_READ
| PROT_EXEC
;
1031 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
1032 uintptr_t start
= 0;
1033 size_t size
= tcg_ctx
->code_gen_buffer_size
;
1036 /* Constrain the position of the buffer based on the host cpu.
1037 Note that these addresses are chosen in concert with the
1038 addresses assigned in the relevant linker script file. */
1039 # if defined(__PIE__) || defined(__PIC__)
1040 /* Don't bother setting a preferred location if we're building
1041 a position-independent executable. We're more likely to get
1042 an address near the main executable if we let the kernel
1043 choose the address. */
1044 # elif defined(__x86_64__) && defined(MAP_32BIT)
1045 /* Force the memory down into low memory with the executable.
1046 Leave the choice of exact location with the kernel. */
1048 /* Cannot expect to map more than 800MB in low memory. */
1049 if (size
> 800u * 1024 * 1024) {
1050 tcg_ctx
->code_gen_buffer_size
= size
= 800u * 1024 * 1024;
1052 # elif defined(__sparc__)
1053 start
= 0x40000000ul
;
1054 # elif defined(__s390x__)
1055 start
= 0x90000000ul
;
1056 # elif defined(__mips__)
1057 # if _MIPS_SIM == _ABI64
1058 start
= 0x128000000ul
;
1060 start
= 0x08000000ul
;
1064 buf
= mmap((void *)start
, size
, prot
, flags
, -1, 0);
1065 if (buf
== MAP_FAILED
) {
1070 if (cross_256mb(buf
, size
)) {
1071 /* Try again, with the original still mapped, to avoid re-acquiring
1072 that 256mb crossing. This time don't specify an address. */
1074 void *buf2
= mmap(NULL
, size
, prot
, flags
, -1, 0);
1075 switch ((int)(buf2
!= MAP_FAILED
)) {
1077 if (!cross_256mb(buf2
, size
)) {
1078 /* Success! Use the new buffer. */
1082 /* Failure. Work with what we had. */
1086 /* Split the original buffer. Free the smaller half. */
1087 buf2
= split_cross_256mb(buf
, size
);
1088 size2
= tcg_ctx
->code_gen_buffer_size
;
1090 munmap(buf
+ size2
, size
- size2
);
1092 munmap(buf
, size
- size2
);
1101 /* Request large pages for the buffer. */
1102 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
1106 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
1108 static inline void code_gen_alloc(size_t tb_size
)
1110 tcg_ctx
->code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
1111 tcg_ctx
->code_gen_buffer
= alloc_code_gen_buffer();
1112 if (tcg_ctx
->code_gen_buffer
== NULL
) {
1113 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
1118 static bool tb_cmp(const void *ap
, const void *bp
)
1120 const TranslationBlock
*a
= ap
;
1121 const TranslationBlock
*b
= bp
;
1123 return a
->pc
== b
->pc
&&
1124 a
->cs_base
== b
->cs_base
&&
1125 a
->flags
== b
->flags
&&
1126 (tb_cflags(a
) & CF_HASH_MASK
) == (tb_cflags(b
) & CF_HASH_MASK
) &&
1127 a
->trace_vcpu_dstate
== b
->trace_vcpu_dstate
&&
1128 a
->page_addr
[0] == b
->page_addr
[0] &&
1129 a
->page_addr
[1] == b
->page_addr
[1];
1132 static void tb_htable_init(void)
1134 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
1136 qht_init(&tb_ctx
.htable
, tb_cmp
, CODE_GEN_HTABLE_SIZE
, mode
);
1139 /* Must be called before using the QEMU cpus. 'tb_size' is the size
1140 (in bytes) allocated to the translation buffer. Zero means default
1142 void tcg_exec_init(unsigned long tb_size
)
1148 code_gen_alloc(tb_size
);
1149 #if defined(CONFIG_SOFTMMU)
1150 /* There's no guest base to take into account, so go ahead and
1151 initialize the prologue now. */
1152 tcg_prologue_init(tcg_ctx
);
1157 * Allocate a new translation block. Flush the translation buffer if
1158 * too many translation blocks or too much generated code.
1160 static TranslationBlock
*tb_alloc(target_ulong pc
)
1162 TranslationBlock
*tb
;
1164 assert_memory_lock();
1166 tb
= tcg_tb_alloc(tcg_ctx
);
1167 if (unlikely(tb
== NULL
)) {
1173 /* call with @p->lock held */
1174 static inline void invalidate_page_bitmap(PageDesc
*p
)
1176 assert_page_locked(p
);
1177 #ifdef CONFIG_SOFTMMU
1178 g_free(p
->code_bitmap
);
1179 p
->code_bitmap
= NULL
;
1180 p
->code_write_count
= 0;
1184 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
1185 static void page_flush_tb_1(int level
, void **lp
)
1195 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1197 pd
[i
].first_tb
= (uintptr_t)NULL
;
1198 invalidate_page_bitmap(pd
+ i
);
1199 page_unlock(&pd
[i
]);
1204 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1205 page_flush_tb_1(level
- 1, pp
+ i
);
1210 static void page_flush_tb(void)
1212 int i
, l1_sz
= v_l1_size
;
1214 for (i
= 0; i
< l1_sz
; i
++) {
1215 page_flush_tb_1(v_l2_levels
, l1_map
+ i
);
1219 static gboolean
tb_host_size_iter(gpointer key
, gpointer value
, gpointer data
)
1221 const TranslationBlock
*tb
= value
;
1222 size_t *size
= data
;
1224 *size
+= tb
->tc
.size
;
1228 /* flush all the translation blocks */
1229 static void do_tb_flush(CPUState
*cpu
, run_on_cpu_data tb_flush_count
)
1232 /* If it is already been done on request of another CPU,
1235 if (tb_ctx
.tb_flush_count
!= tb_flush_count
.host_int
) {
1239 if (DEBUG_TB_FLUSH_GATE
) {
1240 size_t nb_tbs
= tcg_nb_tbs();
1241 size_t host_size
= 0;
1243 tcg_tb_foreach(tb_host_size_iter
, &host_size
);
1244 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
1245 tcg_code_size(), nb_tbs
, nb_tbs
> 0 ? host_size
/ nb_tbs
: 0);
1249 cpu_tb_jmp_cache_clear(cpu
);
1252 qht_reset_size(&tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
1255 tcg_region_reset_all();
1256 /* XXX: flush processor icache at this point if cache flush is
1258 atomic_mb_set(&tb_ctx
.tb_flush_count
, tb_ctx
.tb_flush_count
+ 1);
1264 void tb_flush(CPUState
*cpu
)
1266 if (tcg_enabled()) {
1267 unsigned tb_flush_count
= atomic_mb_read(&tb_ctx
.tb_flush_count
);
1268 async_safe_run_on_cpu(cpu
, do_tb_flush
,
1269 RUN_ON_CPU_HOST_INT(tb_flush_count
));
1274 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
1275 * so in order to prevent bit rot we compile them unconditionally in user-mode,
1276 * and let the optimizer get rid of them by wrapping their user-only callers
1277 * with if (DEBUG_TB_CHECK_GATE).
1279 #ifdef CONFIG_USER_ONLY
1281 static void do_tb_invalidate_check(void *p
, uint32_t hash
, void *userp
)
1283 TranslationBlock
*tb
= p
;
1284 target_ulong addr
= *(target_ulong
*)userp
;
1286 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
1287 printf("ERROR invalidate: address=" TARGET_FMT_lx
1288 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
1292 /* verify that all the pages have correct rights for code
1294 * Called with mmap_lock held.
1296 static void tb_invalidate_check(target_ulong address
)
1298 address
&= TARGET_PAGE_MASK
;
1299 qht_iter(&tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
1302 static void do_tb_page_check(void *p
, uint32_t hash
, void *userp
)
1304 TranslationBlock
*tb
= p
;
1307 flags1
= page_get_flags(tb
->pc
);
1308 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
1309 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
1310 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1311 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
1315 /* verify that all the pages have correct rights for code */
1316 static void tb_page_check(void)
1318 qht_iter(&tb_ctx
.htable
, do_tb_page_check
, NULL
);
1321 #endif /* CONFIG_USER_ONLY */
1324 * user-mode: call with mmap_lock held
1325 * !user-mode: call with @pd->lock held
1327 static inline void tb_page_remove(PageDesc
*pd
, TranslationBlock
*tb
)
1329 TranslationBlock
*tb1
;
1333 assert_page_locked(pd
);
1334 pprev
= &pd
->first_tb
;
1335 PAGE_FOR_EACH_TB(pd
, tb1
, n1
) {
1337 *pprev
= tb1
->page_next
[n1
];
1340 pprev
= &tb1
->page_next
[n1
];
1342 g_assert_not_reached();
1345 /* remove @orig from its @n_orig-th jump list */
1346 static inline void tb_remove_from_jmp_list(TranslationBlock
*orig
, int n_orig
)
1348 uintptr_t ptr
, ptr_locked
;
1349 TranslationBlock
*dest
;
1350 TranslationBlock
*tb
;
1354 /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
1355 ptr
= atomic_or_fetch(&orig
->jmp_dest
[n_orig
], 1);
1356 dest
= (TranslationBlock
*)(ptr
& ~1);
1361 qemu_spin_lock(&dest
->jmp_lock
);
1363 * While acquiring the lock, the jump might have been removed if the
1364 * destination TB was invalidated; check again.
1366 ptr_locked
= atomic_read(&orig
->jmp_dest
[n_orig
]);
1367 if (ptr_locked
!= ptr
) {
1368 qemu_spin_unlock(&dest
->jmp_lock
);
1370 * The only possibility is that the jump was unlinked via
1371 * tb_jump_unlink(dest). Seeing here another destination would be a bug,
1372 * because we set the LSB above.
1374 g_assert(ptr_locked
== 1 && dest
->cflags
& CF_INVALID
);
1378 * We first acquired the lock, and since the destination pointer matches,
1379 * we know for sure that @orig is in the jmp list.
1381 pprev
= &dest
->jmp_list_head
;
1382 TB_FOR_EACH_JMP(dest
, tb
, n
) {
1383 if (tb
== orig
&& n
== n_orig
) {
1384 *pprev
= tb
->jmp_list_next
[n
];
1385 /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
1386 qemu_spin_unlock(&dest
->jmp_lock
);
1389 pprev
= &tb
->jmp_list_next
[n
];
1391 g_assert_not_reached();
1394 /* reset the jump entry 'n' of a TB so that it is not chained to
1396 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
1398 uintptr_t addr
= (uintptr_t)(tb
->tc
.ptr
+ tb
->jmp_reset_offset
[n
]);
1399 tb_set_jmp_target(tb
, n
, addr
);
1402 /* remove any jumps to the TB */
1403 static inline void tb_jmp_unlink(TranslationBlock
*dest
)
1405 TranslationBlock
*tb
;
1408 qemu_spin_lock(&dest
->jmp_lock
);
1410 TB_FOR_EACH_JMP(dest
, tb
, n
) {
1411 tb_reset_jump(tb
, n
);
1412 atomic_and(&tb
->jmp_dest
[n
], (uintptr_t)NULL
| 1);
1413 /* No need to clear the list entry; setting the dest ptr is enough */
1415 dest
->jmp_list_head
= (uintptr_t)NULL
;
1417 qemu_spin_unlock(&dest
->jmp_lock
);
1421 * In user-mode, call with mmap_lock held.
1422 * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
1425 static void do_tb_phys_invalidate(TranslationBlock
*tb
, bool rm_from_page_list
)
1430 tb_page_addr_t phys_pc
;
1432 assert_memory_lock();
1434 /* make sure no further incoming jumps will be chained to this TB */
1435 qemu_spin_lock(&tb
->jmp_lock
);
1436 atomic_set(&tb
->cflags
, tb
->cflags
| CF_INVALID
);
1437 qemu_spin_unlock(&tb
->jmp_lock
);
1439 /* remove the TB from the hash list */
1440 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1441 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb_cflags(tb
) & CF_HASH_MASK
,
1442 tb
->trace_vcpu_dstate
);
1443 if (!(tb
->cflags
& CF_NOCACHE
) &&
1444 !qht_remove(&tb_ctx
.htable
, tb
, h
)) {
1448 /* remove the TB from the page list */
1449 if (rm_from_page_list
) {
1450 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1451 tb_page_remove(p
, tb
);
1452 invalidate_page_bitmap(p
);
1453 if (tb
->page_addr
[1] != -1) {
1454 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1455 tb_page_remove(p
, tb
);
1456 invalidate_page_bitmap(p
);
1460 /* remove the TB from the hash list */
1461 h
= tb_jmp_cache_hash_func(tb
->pc
);
1463 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1464 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1468 /* suppress this TB from the two jump lists */
1469 tb_remove_from_jmp_list(tb
, 0);
1470 tb_remove_from_jmp_list(tb
, 1);
1472 /* suppress any remaining jumps to this TB */
1475 atomic_set(&tcg_ctx
->tb_phys_invalidate_count
,
1476 tcg_ctx
->tb_phys_invalidate_count
+ 1);
1479 static void tb_phys_invalidate__locked(TranslationBlock
*tb
)
1481 do_tb_phys_invalidate(tb
, true);
1484 /* invalidate one TB
1486 * Called with mmap_lock held in user-mode.
1488 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
1490 if (page_addr
== -1 && tb
->page_addr
[0] != -1) {
1492 do_tb_phys_invalidate(tb
, true);
1495 do_tb_phys_invalidate(tb
, false);
1499 #ifdef CONFIG_SOFTMMU
1500 /* call with @p->lock held */
1501 static void build_page_bitmap(PageDesc
*p
)
1503 int n
, tb_start
, tb_end
;
1504 TranslationBlock
*tb
;
1506 assert_page_locked(p
);
1507 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1509 PAGE_FOR_EACH_TB(p
, tb
, n
) {
1510 /* NOTE: this is subtle as a TB may span two physical pages */
1512 /* NOTE: tb_end may be after the end of the page, but
1513 it is not a problem */
1514 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1515 tb_end
= tb_start
+ tb
->size
;
1516 if (tb_end
> TARGET_PAGE_SIZE
) {
1517 tb_end
= TARGET_PAGE_SIZE
;
1521 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1523 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1528 /* add the tb in the target page and protect it if necessary
1530 * Called with mmap_lock held for user-mode emulation.
1531 * Called with @p->lock held in !user-mode.
1533 static inline void tb_page_add(PageDesc
*p
, TranslationBlock
*tb
,
1534 unsigned int n
, tb_page_addr_t page_addr
)
1536 #ifndef CONFIG_USER_ONLY
1537 bool page_already_protected
;
1540 assert_page_locked(p
);
1542 tb
->page_addr
[n
] = page_addr
;
1543 tb
->page_next
[n
] = p
->first_tb
;
1544 #ifndef CONFIG_USER_ONLY
1545 page_already_protected
= p
->first_tb
!= (uintptr_t)NULL
;
1547 p
->first_tb
= (uintptr_t)tb
| n
;
1548 invalidate_page_bitmap(p
);
1550 #if defined(CONFIG_USER_ONLY)
1551 if (p
->flags
& PAGE_WRITE
) {
1556 /* force the host page as non writable (writes will have a
1557 page fault + mprotect overhead) */
1558 page_addr
&= qemu_host_page_mask
;
1560 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1561 addr
+= TARGET_PAGE_SIZE
) {
1563 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1568 p2
->flags
&= ~PAGE_WRITE
;
1570 mprotect(g2h(page_addr
), qemu_host_page_size
,
1571 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1572 if (DEBUG_TB_INVALIDATE_GATE
) {
1573 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT
"\n", page_addr
);
1577 /* if some code is already present, then the pages are already
1578 protected. So we handle the case where only the first TB is
1579 allocated in a physical page */
1580 if (!page_already_protected
) {
1581 tlb_protect_code(page_addr
);
1586 /* add a new TB and link it to the physical page tables. phys_page2 is
1587 * (-1) to indicate that only one page contains the TB.
1589 * Called with mmap_lock held for user-mode emulation.
1591 * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
1592 * Note that in !user-mode, another thread might have already added a TB
1593 * for the same block of guest code that @tb corresponds to. In that case,
1594 * the caller should discard the original @tb, and use instead the returned TB.
1596 static TranslationBlock
*
1597 tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1598 tb_page_addr_t phys_page2
)
1601 PageDesc
*p2
= NULL
;
1603 assert_memory_lock();
1605 if (phys_pc
== -1) {
1607 * If the TB is not associated with a physical RAM page then
1608 * it must be a temporary one-insn TB, and we have nothing to do
1609 * except fill in the page_addr[] fields.
1611 assert(tb
->cflags
& CF_NOCACHE
);
1612 tb
->page_addr
[0] = tb
->page_addr
[1] = -1;
1617 * Add the TB to the page list, acquiring first the pages's locks.
1618 * We keep the locks held until after inserting the TB in the hash table,
1619 * so that if the insertion fails we know for sure that the TBs are still
1620 * in the page descriptors.
1621 * Note that inserting into the hash table first isn't an option, since
1622 * we can only insert TBs that are fully initialized.
1624 page_lock_pair(&p
, phys_pc
, &p2
, phys_page2
, 1);
1625 tb_page_add(p
, tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1627 tb_page_add(p2
, tb
, 1, phys_page2
);
1629 tb
->page_addr
[1] = -1;
1632 if (!(tb
->cflags
& CF_NOCACHE
)) {
1633 void *existing_tb
= NULL
;
1636 /* add in the hash table */
1637 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->cflags
& CF_HASH_MASK
,
1638 tb
->trace_vcpu_dstate
);
1639 qht_insert(&tb_ctx
.htable
, tb
, h
, &existing_tb
);
1641 /* remove TB from the page(s) if we couldn't insert it */
1642 if (unlikely(existing_tb
)) {
1643 tb_page_remove(p
, tb
);
1644 invalidate_page_bitmap(p
);
1646 tb_page_remove(p2
, tb
);
1647 invalidate_page_bitmap(p2
);
1653 if (p2
&& p2
!= p
) {
1658 #ifdef CONFIG_USER_ONLY
1659 if (DEBUG_TB_CHECK_GATE
) {
1666 /* Called with mmap_lock held for user mode emulation. */
1667 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1668 target_ulong pc
, target_ulong cs_base
,
1669 uint32_t flags
, int cflags
)
1671 CPUArchState
*env
= cpu
->env_ptr
;
1672 TranslationBlock
*tb
, *existing_tb
;
1673 tb_page_addr_t phys_pc
, phys_page2
;
1674 target_ulong virt_page2
;
1675 tcg_insn_unit
*gen_code_buf
;
1676 int gen_code_size
, search_size
;
1677 #ifdef CONFIG_PROFILER
1678 TCGProfile
*prof
= &tcg_ctx
->prof
;
1681 assert_memory_lock();
1683 phys_pc
= get_page_addr_code(env
, pc
);
1685 if (phys_pc
== -1) {
1686 /* Generate a temporary TB with 1 insn in it */
1687 cflags
&= ~CF_COUNT_MASK
;
1688 cflags
|= CF_NOCACHE
| 1;
1691 cflags
&= ~CF_CLUSTER_MASK
;
1692 cflags
|= cpu
->cluster_index
<< CF_CLUSTER_SHIFT
;
1696 if (unlikely(!tb
)) {
1697 /* flush must be done */
1700 /* Make the execution loop process the flush as soon as possible. */
1701 cpu
->exception_index
= EXCP_INTERRUPT
;
1705 gen_code_buf
= tcg_ctx
->code_gen_ptr
;
1706 tb
->tc
.ptr
= gen_code_buf
;
1708 tb
->cs_base
= cs_base
;
1710 tb
->cflags
= cflags
;
1711 tb
->trace_vcpu_dstate
= *cpu
->trace_dstate
;
1712 tcg_ctx
->tb_cflags
= cflags
;
1714 #ifdef CONFIG_PROFILER
1715 /* includes aborted translations because of exceptions */
1716 atomic_set(&prof
->tb_count1
, prof
->tb_count1
+ 1);
1717 ti
= profile_getclock();
1720 tcg_func_start(tcg_ctx
);
1722 tcg_ctx
->cpu
= ENV_GET_CPU(env
);
1723 gen_intermediate_code(cpu
, tb
);
1724 tcg_ctx
->cpu
= NULL
;
1726 trace_translate_block(tb
, tb
->pc
, tb
->tc
.ptr
);
1728 /* generate machine code */
1729 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1730 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1731 tcg_ctx
->tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1732 if (TCG_TARGET_HAS_direct_jump
) {
1733 tcg_ctx
->tb_jmp_insn_offset
= tb
->jmp_target_arg
;
1734 tcg_ctx
->tb_jmp_target_addr
= NULL
;
1736 tcg_ctx
->tb_jmp_insn_offset
= NULL
;
1737 tcg_ctx
->tb_jmp_target_addr
= tb
->jmp_target_arg
;
1740 #ifdef CONFIG_PROFILER
1741 atomic_set(&prof
->tb_count
, prof
->tb_count
+ 1);
1742 atomic_set(&prof
->interm_time
, prof
->interm_time
+ profile_getclock() - ti
);
1743 ti
= profile_getclock();
1746 /* ??? Overflow could be handled better here. In particular, we
1747 don't need to re-do gen_intermediate_code, nor should we re-do
1748 the tcg optimization currently hidden inside tcg_gen_code. All
1749 that should be required is to flush the TBs, allocate a new TB,
1750 re-initialize it per above, and re-do the actual code generation. */
1751 gen_code_size
= tcg_gen_code(tcg_ctx
, tb
);
1752 if (unlikely(gen_code_size
< 0)) {
1753 goto buffer_overflow
;
1755 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1756 if (unlikely(search_size
< 0)) {
1757 goto buffer_overflow
;
1759 tb
->tc
.size
= gen_code_size
;
1761 #ifdef CONFIG_PROFILER
1762 atomic_set(&prof
->code_time
, prof
->code_time
+ profile_getclock() - ti
);
1763 atomic_set(&prof
->code_in_len
, prof
->code_in_len
+ tb
->size
);
1764 atomic_set(&prof
->code_out_len
, prof
->code_out_len
+ gen_code_size
);
1765 atomic_set(&prof
->search_out_len
, prof
->search_out_len
+ search_size
);
1769 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1770 qemu_log_in_addr_range(tb
->pc
)) {
1772 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1773 if (tcg_ctx
->data_gen_ptr
) {
1774 size_t code_size
= tcg_ctx
->data_gen_ptr
- tb
->tc
.ptr
;
1775 size_t data_size
= gen_code_size
- code_size
;
1778 log_disas(tb
->tc
.ptr
, code_size
);
1780 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1781 if (sizeof(tcg_target_ulong
) == 8) {
1782 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1783 (uintptr_t)tcg_ctx
->data_gen_ptr
+ i
,
1784 *(uint64_t *)(tcg_ctx
->data_gen_ptr
+ i
));
1786 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1787 (uintptr_t)tcg_ctx
->data_gen_ptr
+ i
,
1788 *(uint32_t *)(tcg_ctx
->data_gen_ptr
+ i
));
1792 log_disas(tb
->tc
.ptr
, gen_code_size
);
1800 atomic_set(&tcg_ctx
->code_gen_ptr
, (void *)
1801 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1804 /* init jump list */
1805 qemu_spin_init(&tb
->jmp_lock
);
1806 tb
->jmp_list_head
= (uintptr_t)NULL
;
1807 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1808 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1809 tb
->jmp_dest
[0] = (uintptr_t)NULL
;
1810 tb
->jmp_dest
[1] = (uintptr_t)NULL
;
1812 /* init original jump addresses which have been set during tcg_gen_code() */
1813 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1814 tb_reset_jump(tb
, 0);
1816 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1817 tb_reset_jump(tb
, 1);
1820 /* check next page if needed */
1821 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1823 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1824 phys_page2
= get_page_addr_code(env
, virt_page2
);
1827 * No explicit memory barrier is required -- tb_link_page() makes the
1828 * TB visible in a consistent state.
1830 existing_tb
= tb_link_page(tb
, phys_pc
, phys_page2
);
1831 /* if the TB already exists, discard what we just translated */
1832 if (unlikely(existing_tb
!= tb
)) {
1833 uintptr_t orig_aligned
= (uintptr_t)gen_code_buf
;
1835 orig_aligned
-= ROUND_UP(sizeof(*tb
), qemu_icache_linesize
);
1836 atomic_set(&tcg_ctx
->code_gen_ptr
, (void *)orig_aligned
);
1844 * @p must be non-NULL.
1845 * user-mode: call with mmap_lock held.
1846 * !user-mode: call with all @pages locked.
1849 tb_invalidate_phys_page_range__locked(struct page_collection
*pages
,
1850 PageDesc
*p
, tb_page_addr_t start
,
1852 int is_cpu_write_access
)
1854 TranslationBlock
*tb
;
1855 tb_page_addr_t tb_start
, tb_end
;
1857 #ifdef TARGET_HAS_PRECISE_SMC
1858 CPUState
*cpu
= current_cpu
;
1859 CPUArchState
*env
= NULL
;
1860 int current_tb_not_found
= is_cpu_write_access
;
1861 TranslationBlock
*current_tb
= NULL
;
1862 int current_tb_modified
= 0;
1863 target_ulong current_pc
= 0;
1864 target_ulong current_cs_base
= 0;
1865 uint32_t current_flags
= 0;
1866 #endif /* TARGET_HAS_PRECISE_SMC */
1868 assert_page_locked(p
);
1870 #if defined(TARGET_HAS_PRECISE_SMC)
1876 /* we remove all the TBs in the range [start, end[ */
1877 /* XXX: see if in some cases it could be faster to invalidate all
1879 PAGE_FOR_EACH_TB(p
, tb
, n
) {
1880 assert_page_locked(p
);
1881 /* NOTE: this is subtle as a TB may span two physical pages */
1883 /* NOTE: tb_end may be after the end of the page, but
1884 it is not a problem */
1885 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1886 tb_end
= tb_start
+ tb
->size
;
1888 tb_start
= tb
->page_addr
[1];
1889 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1891 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1892 #ifdef TARGET_HAS_PRECISE_SMC
1893 if (current_tb_not_found
) {
1894 current_tb_not_found
= 0;
1896 if (cpu
->mem_io_pc
) {
1897 /* now we have a real cpu fault */
1898 current_tb
= tcg_tb_lookup(cpu
->mem_io_pc
);
1901 if (current_tb
== tb
&&
1902 (tb_cflags(current_tb
) & CF_COUNT_MASK
) != 1) {
1903 /* If we are modifying the current TB, we must stop
1904 its execution. We could be more precise by checking
1905 that the modification is after the current PC, but it
1906 would require a specialized function to partially
1907 restore the CPU state */
1909 current_tb_modified
= 1;
1910 cpu_restore_state_from_tb(cpu
, current_tb
,
1911 cpu
->mem_io_pc
, true);
1912 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1915 #endif /* TARGET_HAS_PRECISE_SMC */
1916 tb_phys_invalidate__locked(tb
);
1919 #if !defined(CONFIG_USER_ONLY)
1920 /* if no code remaining, no need to continue to use slow writes */
1922 invalidate_page_bitmap(p
);
1923 tlb_unprotect_code(start
);
1926 #ifdef TARGET_HAS_PRECISE_SMC
1927 if (current_tb_modified
) {
1928 page_collection_unlock(pages
);
1929 /* Force execution of one insn next time. */
1930 cpu
->cflags_next_tb
= 1 | curr_cflags();
1932 cpu_loop_exit_noexc(cpu
);
1938 * Invalidate all TBs which intersect with the target physical address range
1939 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1940 * 'is_cpu_write_access' should be true if called from a real cpu write
1941 * access: the virtual CPU will exit the current TB if code is modified inside
1944 * Called with mmap_lock held for user-mode emulation
1946 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1947 int is_cpu_write_access
)
1949 struct page_collection
*pages
;
1952 assert_memory_lock();
1954 p
= page_find(start
>> TARGET_PAGE_BITS
);
1958 pages
= page_collection_lock(start
, end
);
1959 tb_invalidate_phys_page_range__locked(pages
, p
, start
, end
,
1960 is_cpu_write_access
);
1961 page_collection_unlock(pages
);
1965 * Invalidate all TBs which intersect with the target physical address range
1966 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1967 * 'is_cpu_write_access' should be true if called from a real cpu write
1968 * access: the virtual CPU will exit the current TB if code is modified inside
1971 * Called with mmap_lock held for user-mode emulation.
1973 #ifdef CONFIG_SOFTMMU
1974 void tb_invalidate_phys_range(ram_addr_t start
, ram_addr_t end
)
1976 void tb_invalidate_phys_range(target_ulong start
, target_ulong end
)
1979 struct page_collection
*pages
;
1980 tb_page_addr_t next
;
1982 assert_memory_lock();
1984 pages
= page_collection_lock(start
, end
);
1985 for (next
= (start
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
;
1987 start
= next
, next
+= TARGET_PAGE_SIZE
) {
1988 PageDesc
*pd
= page_find(start
>> TARGET_PAGE_BITS
);
1989 tb_page_addr_t bound
= MIN(next
, end
);
1994 tb_invalidate_phys_page_range__locked(pages
, pd
, start
, bound
, 0);
1996 page_collection_unlock(pages
);
1999 #ifdef CONFIG_SOFTMMU
2000 /* len must be <= 8 and start must be a multiple of len.
2001 * Called via softmmu_template.h when code areas are written to with
2002 * iothread mutex not held.
2004 * Call with all @pages in the range [@start, @start + len[ locked.
2006 void tb_invalidate_phys_page_fast(struct page_collection
*pages
,
2007 tb_page_addr_t start
, int len
)
2011 assert_memory_lock();
2013 p
= page_find(start
>> TARGET_PAGE_BITS
);
2018 assert_page_locked(p
);
2019 if (!p
->code_bitmap
&&
2020 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
2021 build_page_bitmap(p
);
2023 if (p
->code_bitmap
) {
2027 nr
= start
& ~TARGET_PAGE_MASK
;
2028 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
2029 if (b
& ((1 << len
) - 1)) {
2034 tb_invalidate_phys_page_range__locked(pages
, p
, start
, start
+ len
, 1);
2038 /* Called with mmap_lock held. If pc is not 0 then it indicates the
2039 * host PC of the faulting store instruction that caused this invalidate.
2040 * Returns true if the caller needs to abort execution of the current
2041 * TB (because it was modified by this store and the guest CPU has
2042 * precise-SMC semantics).
2044 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
2046 TranslationBlock
*tb
;
2049 #ifdef TARGET_HAS_PRECISE_SMC
2050 TranslationBlock
*current_tb
= NULL
;
2051 CPUState
*cpu
= current_cpu
;
2052 CPUArchState
*env
= NULL
;
2053 int current_tb_modified
= 0;
2054 target_ulong current_pc
= 0;
2055 target_ulong current_cs_base
= 0;
2056 uint32_t current_flags
= 0;
2059 assert_memory_lock();
2061 addr
&= TARGET_PAGE_MASK
;
2062 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2067 #ifdef TARGET_HAS_PRECISE_SMC
2068 if (p
->first_tb
&& pc
!= 0) {
2069 current_tb
= tcg_tb_lookup(pc
);
2075 assert_page_locked(p
);
2076 PAGE_FOR_EACH_TB(p
, tb
, n
) {
2077 #ifdef TARGET_HAS_PRECISE_SMC
2078 if (current_tb
== tb
&&
2079 (tb_cflags(current_tb
) & CF_COUNT_MASK
) != 1) {
2080 /* If we are modifying the current TB, we must stop
2081 its execution. We could be more precise by checking
2082 that the modification is after the current PC, but it
2083 would require a specialized function to partially
2084 restore the CPU state */
2086 current_tb_modified
= 1;
2087 cpu_restore_state_from_tb(cpu
, current_tb
, pc
, true);
2088 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
2091 #endif /* TARGET_HAS_PRECISE_SMC */
2092 tb_phys_invalidate(tb
, addr
);
2094 p
->first_tb
= (uintptr_t)NULL
;
2095 #ifdef TARGET_HAS_PRECISE_SMC
2096 if (current_tb_modified
) {
2097 /* Force execution of one insn next time. */
2098 cpu
->cflags_next_tb
= 1 | curr_cflags();
2107 /* user-mode: call with mmap_lock held */
2108 void tb_check_watchpoint(CPUState
*cpu
)
2110 TranslationBlock
*tb
;
2112 assert_memory_lock();
2114 tb
= tcg_tb_lookup(cpu
->mem_io_pc
);
2116 /* We can use retranslation to find the PC. */
2117 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
, true);
2118 tb_phys_invalidate(tb
, -1);
2120 /* The exception probably happened in a helper. The CPU state should
2121 have been saved before calling it. Fetch the PC from there. */
2122 CPUArchState
*env
= cpu
->env_ptr
;
2123 target_ulong pc
, cs_base
;
2124 tb_page_addr_t addr
;
2127 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
2128 addr
= get_page_addr_code(env
, pc
);
2130 tb_invalidate_phys_range(addr
, addr
+ 1);
2135 #ifndef CONFIG_USER_ONLY
2136 /* in deterministic execution mode, instructions doing device I/Os
2137 * must be at the end of the TB.
2139 * Called by softmmu_template.h, with iothread mutex not held.
2141 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
2143 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
2144 CPUArchState
*env
= cpu
->env_ptr
;
2146 TranslationBlock
*tb
;
2149 tb
= tcg_tb_lookup(retaddr
);
2151 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
2154 cpu_restore_state_from_tb(cpu
, tb
, retaddr
, true);
2156 /* On MIPS and SH, delay slot instructions can only be restarted if
2157 they were already the first instruction in the TB. If this is not
2158 the first instruction in a TB then re-execute the preceding
2161 #if defined(TARGET_MIPS)
2162 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0
2163 && env
->active_tc
.PC
!= tb
->pc
) {
2164 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
2165 cpu
->icount_decr
.u16
.low
++;
2166 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
2169 #elif defined(TARGET_SH4)
2170 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
2171 && env
->pc
!= tb
->pc
) {
2173 cpu
->icount_decr
.u16
.low
++;
2174 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
2179 /* Generate a new TB executing the I/O insn. */
2180 cpu
->cflags_next_tb
= curr_cflags() | CF_LAST_IO
| n
;
2182 if (tb_cflags(tb
) & CF_NOCACHE
) {
2184 /* Invalidate original TB if this TB was generated in
2185 * cpu_exec_nocache() */
2186 tb_phys_invalidate(tb
->orig_tb
, -1);
2191 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
2192 * the first in the TB) then we end up generating a whole new TB and
2193 * repeating the fault, which is horribly inefficient.
2194 * Better would be to execute just this insn uncached, or generate a
2197 cpu_loop_exit_noexc(cpu
);
2200 static void tb_jmp_cache_clear_page(CPUState
*cpu
, target_ulong page_addr
)
2202 unsigned int i
, i0
= tb_jmp_cache_hash_page(page_addr
);
2204 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
2205 atomic_set(&cpu
->tb_jmp_cache
[i0
+ i
], NULL
);
2209 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
2211 /* Discard jump cache entries for any tb which might potentially
2212 overlap the flushed page. */
2213 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
2214 tb_jmp_cache_clear_page(cpu
, addr
);
2217 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
2218 struct qht_stats hst
)
2220 uint32_t hgram_opts
;
2224 if (!hst
.head_buckets
) {
2227 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
2228 hst
.used_head_buckets
, hst
.head_buckets
,
2229 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
2231 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
2232 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
2233 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
2234 hgram_opts
|= QDIST_PR_NODECIMAL
;
2236 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
2237 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
2238 qdist_avg(&hst
.occupancy
) * 100, hgram
);
2241 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
2242 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
2243 if (hgram_bins
> 10) {
2247 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
2249 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
2250 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
2251 qdist_avg(&hst
.chain
), hgram
);
2255 struct tb_tree_stats
{
2259 size_t max_target_size
;
2260 size_t direct_jmp_count
;
2261 size_t direct_jmp2_count
;
2265 static gboolean
tb_tree_stats_iter(gpointer key
, gpointer value
, gpointer data
)
2267 const TranslationBlock
*tb
= value
;
2268 struct tb_tree_stats
*tst
= data
;
2271 tst
->host_size
+= tb
->tc
.size
;
2272 tst
->target_size
+= tb
->size
;
2273 if (tb
->size
> tst
->max_target_size
) {
2274 tst
->max_target_size
= tb
->size
;
2276 if (tb
->page_addr
[1] != -1) {
2279 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
2280 tst
->direct_jmp_count
++;
2281 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
2282 tst
->direct_jmp2_count
++;
2288 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
2290 struct tb_tree_stats tst
= {};
2291 struct qht_stats hst
;
2292 size_t nb_tbs
, flush_full
, flush_part
, flush_elide
;
2294 tcg_tb_foreach(tb_tree_stats_iter
, &tst
);
2295 nb_tbs
= tst
.nb_tbs
;
2296 /* XXX: avoid using doubles ? */
2297 cpu_fprintf(f
, "Translation buffer state:\n");
2299 * Report total code size including the padding and TB structs;
2300 * otherwise users might think "-tb-size" is not honoured.
2301 * For avg host size we use the precise numbers from tb_tree_stats though.
2303 cpu_fprintf(f
, "gen code size %zu/%zu\n",
2304 tcg_code_size(), tcg_code_capacity());
2305 cpu_fprintf(f
, "TB count %zu\n", nb_tbs
);
2306 cpu_fprintf(f
, "TB avg target size %zu max=%zu bytes\n",
2307 nb_tbs
? tst
.target_size
/ nb_tbs
: 0,
2308 tst
.max_target_size
);
2309 cpu_fprintf(f
, "TB avg host size %zu bytes (expansion ratio: %0.1f)\n",
2310 nb_tbs
? tst
.host_size
/ nb_tbs
: 0,
2311 tst
.target_size
? (double)tst
.host_size
/ tst
.target_size
: 0);
2312 cpu_fprintf(f
, "cross page TB count %zu (%zu%%)\n", tst
.cross_page
,
2313 nb_tbs
? (tst
.cross_page
* 100) / nb_tbs
: 0);
2314 cpu_fprintf(f
, "direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n",
2315 tst
.direct_jmp_count
,
2316 nb_tbs
? (tst
.direct_jmp_count
* 100) / nb_tbs
: 0,
2317 tst
.direct_jmp2_count
,
2318 nb_tbs
? (tst
.direct_jmp2_count
* 100) / nb_tbs
: 0);
2320 qht_statistics_init(&tb_ctx
.htable
, &hst
);
2321 print_qht_statistics(f
, cpu_fprintf
, hst
);
2322 qht_statistics_destroy(&hst
);
2324 cpu_fprintf(f
, "\nStatistics:\n");
2325 cpu_fprintf(f
, "TB flush count %u\n",
2326 atomic_read(&tb_ctx
.tb_flush_count
));
2327 cpu_fprintf(f
, "TB invalidate count %zu\n", tcg_tb_phys_invalidate_count());
2329 tlb_flush_counts(&flush_full
, &flush_part
, &flush_elide
);
2330 cpu_fprintf(f
, "TLB full flushes %zu\n", flush_full
);
2331 cpu_fprintf(f
, "TLB partial flushes %zu\n", flush_part
);
2332 cpu_fprintf(f
, "TLB elided flushes %zu\n", flush_elide
);
2333 tcg_dump_info(f
, cpu_fprintf
);
2336 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
2338 tcg_dump_op_count(f
, cpu_fprintf
);
2341 #else /* CONFIG_USER_ONLY */
2343 void cpu_interrupt(CPUState
*cpu
, int mask
)
2345 g_assert(qemu_mutex_iothread_locked());
2346 cpu
->interrupt_request
|= mask
;
2347 atomic_set(&cpu
->icount_decr
.u16
.high
, -1);
2351 * Walks guest process memory "regions" one by one
2352 * and calls callback function 'fn' for each region.
2354 struct walk_memory_regions_data
{
2355 walk_memory_regions_fn fn
;
2361 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2362 target_ulong end
, int new_prot
)
2364 if (data
->start
!= -1u) {
2365 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2371 data
->start
= (new_prot
? end
: -1u);
2372 data
->prot
= new_prot
;
2377 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2378 target_ulong base
, int level
, void **lp
)
2384 return walk_memory_regions_end(data
, base
, 0);
2390 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2391 int prot
= pd
[i
].flags
;
2393 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2394 if (prot
!= data
->prot
) {
2395 rc
= walk_memory_regions_end(data
, pa
, prot
);
2404 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2405 pa
= base
| ((target_ulong
)i
<<
2406 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
2407 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2417 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2419 struct walk_memory_regions_data data
;
2420 uintptr_t i
, l1_sz
= v_l1_size
;
2427 for (i
= 0; i
< l1_sz
; i
++) {
2428 target_ulong base
= i
<< (v_l1_shift
+ TARGET_PAGE_BITS
);
2429 int rc
= walk_memory_regions_1(&data
, base
, v_l2_levels
, l1_map
+ i
);
2435 return walk_memory_regions_end(&data
, 0, 0);
2438 static int dump_region(void *priv
, target_ulong start
,
2439 target_ulong end
, unsigned long prot
)
2441 FILE *f
= (FILE *)priv
;
2443 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
2444 " "TARGET_FMT_lx
" %c%c%c\n",
2445 start
, end
, end
- start
,
2446 ((prot
& PAGE_READ
) ? 'r' : '-'),
2447 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2448 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2453 /* dump memory mappings */
2454 void page_dump(FILE *f
)
2456 const int length
= sizeof(target_ulong
) * 2;
2457 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
2458 length
, "start", length
, "end", length
, "size", "prot");
2459 walk_memory_regions(f
, dump_region
);
2462 int page_get_flags(target_ulong address
)
2466 p
= page_find(address
>> TARGET_PAGE_BITS
);
2473 /* Modify the flags of a page and invalidate the code if necessary.
2474 The flag PAGE_WRITE_ORG is positioned automatically depending
2475 on PAGE_WRITE. The mmap_lock should already be held. */
2476 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2478 target_ulong addr
, len
;
2480 /* This function should never be called with addresses outside the
2481 guest address space. If this assert fires, it probably indicates
2482 a missing call to h2g_valid. */
2483 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2484 assert(end
<= ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2486 assert(start
< end
);
2487 assert_memory_lock();
2489 start
= start
& TARGET_PAGE_MASK
;
2490 end
= TARGET_PAGE_ALIGN(end
);
2492 if (flags
& PAGE_WRITE
) {
2493 flags
|= PAGE_WRITE_ORG
;
2496 for (addr
= start
, len
= end
- start
;
2498 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2499 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2501 /* If the write protection bit is set, then we invalidate
2503 if (!(p
->flags
& PAGE_WRITE
) &&
2504 (flags
& PAGE_WRITE
) &&
2506 tb_invalidate_phys_page(addr
, 0);
2512 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2518 /* This function should never be called with addresses outside the
2519 guest address space. If this assert fires, it probably indicates
2520 a missing call to h2g_valid. */
2521 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2522 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2528 if (start
+ len
- 1 < start
) {
2529 /* We've wrapped around. */
2533 /* must do before we loose bits in the next step */
2534 end
= TARGET_PAGE_ALIGN(start
+ len
);
2535 start
= start
& TARGET_PAGE_MASK
;
2537 for (addr
= start
, len
= end
- start
;
2539 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2540 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2544 if (!(p
->flags
& PAGE_VALID
)) {
2548 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
2551 if (flags
& PAGE_WRITE
) {
2552 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2555 /* unprotect the page if it was put read-only because it
2556 contains translated code */
2557 if (!(p
->flags
& PAGE_WRITE
)) {
2558 if (!page_unprotect(addr
, 0)) {
2567 /* called from signal handler: invalidate the code and unprotect the
2568 * page. Return 0 if the fault was not handled, 1 if it was handled,
2569 * and 2 if it was handled but the caller must cause the TB to be
2570 * immediately exited. (We can only return 2 if the 'pc' argument is
2573 int page_unprotect(target_ulong address
, uintptr_t pc
)
2576 bool current_tb_invalidated
;
2578 target_ulong host_start
, host_end
, addr
;
2580 /* Technically this isn't safe inside a signal handler. However we
2581 know this only ever happens in a synchronous SEGV handler, so in
2582 practice it seems to be ok. */
2585 p
= page_find(address
>> TARGET_PAGE_BITS
);
2591 /* if the page was really writable, then we change its
2592 protection back to writable */
2593 if (p
->flags
& PAGE_WRITE_ORG
) {
2594 current_tb_invalidated
= false;
2595 if (p
->flags
& PAGE_WRITE
) {
2596 /* If the page is actually marked WRITE then assume this is because
2597 * this thread raced with another one which got here first and
2598 * set the page to PAGE_WRITE and did the TB invalidate for us.
2600 #ifdef TARGET_HAS_PRECISE_SMC
2601 TranslationBlock
*current_tb
= tcg_tb_lookup(pc
);
2603 current_tb_invalidated
= tb_cflags(current_tb
) & CF_INVALID
;
2607 host_start
= address
& qemu_host_page_mask
;
2608 host_end
= host_start
+ qemu_host_page_size
;
2611 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2612 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2613 p
->flags
|= PAGE_WRITE
;
2616 /* and since the content will be modified, we must invalidate
2617 the corresponding translated code. */
2618 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2619 #ifdef CONFIG_USER_ONLY
2620 if (DEBUG_TB_CHECK_GATE
) {
2621 tb_invalidate_check(addr
);
2625 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2629 /* If current TB was invalidated return to main loop */
2630 return current_tb_invalidated
? 2 : 1;
2635 #endif /* CONFIG_USER_ONLY */
2637 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2638 void tcg_flush_softmmu_tlb(CPUState
*cs
)
2640 #ifdef CONFIG_SOFTMMU