4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
21 #include "qemu-common.h"
22 #define NO_CPU_IO_DEFS
25 #include "disas/disas.h"
26 #include "exec/exec-all.h"
28 #if defined(CONFIG_USER_ONLY)
30 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
31 #include <sys/param.h>
32 #if __FreeBSD_version >= 700104
33 #define HAVE_KINFO_GETVMMAP
34 #define sigqueue sigqueue_freebsd /* avoid redefinition */
36 #include <machine/profile.h>
45 #include "exec/ram_addr.h"
48 #include "exec/cputlb.h"
49 #include "exec/tb-hash.h"
50 #include "translate-all.h"
51 #include "qemu/bitmap.h"
52 #include "qemu/error-report.h"
53 #include "qemu/qemu-print.h"
54 #include "qemu/timer.h"
55 #include "qemu/main-loop.h"
57 #include "sysemu/cpus.h"
59 /* #define DEBUG_TB_INVALIDATE */
60 /* #define DEBUG_TB_FLUSH */
61 /* make various TB consistency checks */
62 /* #define DEBUG_TB_CHECK */
64 #ifdef DEBUG_TB_INVALIDATE
65 #define DEBUG_TB_INVALIDATE_GATE 1
67 #define DEBUG_TB_INVALIDATE_GATE 0
71 #define DEBUG_TB_FLUSH_GATE 1
73 #define DEBUG_TB_FLUSH_GATE 0
76 #if !defined(CONFIG_USER_ONLY)
77 /* TB consistency checks only implemented for usermode emulation. */
82 #define DEBUG_TB_CHECK_GATE 1
84 #define DEBUG_TB_CHECK_GATE 0
87 /* Access to the various translations structures need to be serialised via locks
89 * In user-mode emulation access to the memory related structures are protected
91 * In !user-mode we use per-page locks.
94 #define assert_memory_lock()
96 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
99 #define SMC_BITMAP_USE_THRESHOLD 10
101 typedef struct PageDesc
{
102 /* list of TBs intersecting this ram page */
104 #ifdef CONFIG_SOFTMMU
105 /* in order to optimize self modifying code, we count the number
106 of lookups we do to a given page to use a bitmap */
107 unsigned long *code_bitmap
;
108 unsigned int code_write_count
;
112 #ifndef CONFIG_USER_ONLY
118 * struct page_entry - page descriptor entry
119 * @pd: pointer to the &struct PageDesc of the page this entry represents
120 * @index: page index of the page
121 * @locked: whether the page is locked
123 * This struct helps us keep track of the locked state of a page, without
124 * bloating &struct PageDesc.
126 * A page lock protects accesses to all fields of &struct PageDesc.
128 * See also: &struct page_collection.
132 tb_page_addr_t index
;
137 * struct page_collection - tracks a set of pages (i.e. &struct page_entry's)
138 * @tree: Binary search tree (BST) of the pages, with key == page index
139 * @max: Pointer to the page in @tree with the highest page index
141 * To avoid deadlock we lock pages in ascending order of page index.
142 * When operating on a set of pages, we need to keep track of them so that
143 * we can lock them in order and also unlock them later. For this we collect
144 * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the
145 * @tree implementation we use does not provide an O(1) operation to obtain the
146 * highest-ranked element, we use @max to keep track of the inserted page
147 * with the highest index. This is valuable because if a page is not in
148 * the tree and its index is higher than @max's, then we can lock it
149 * without breaking the locking order rule.
151 * Note on naming: 'struct page_set' would be shorter, but we already have a few
152 * page_set_*() helpers, so page_collection is used instead to avoid confusion.
154 * See also: page_collection_lock().
156 struct page_collection
{
158 struct page_entry
*max
;
161 /* list iterators for lists of tagged pointers in TranslationBlock */
162 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \
163 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
164 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
165 tb = (TranslationBlock *)((uintptr_t)tb & ~1))
167 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
168 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
170 #define TB_FOR_EACH_JMP(head_tb, tb, n) \
171 TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
173 /* In system mode we want L1_MAP to be based on ram offsets,
174 while in user mode we want it to be based on virtual addresses. */
175 #if !defined(CONFIG_USER_ONLY)
176 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
177 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
179 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
182 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
185 /* Size of the L2 (and L3, etc) page tables. */
187 #define V_L2_SIZE (1 << V_L2_BITS)
189 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
190 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS
>
191 sizeof_field(TranslationBlock
, trace_vcpu_dstate
)
195 * L1 Mapping properties
197 static int v_l1_size
;
198 static int v_l1_shift
;
199 static int v_l2_levels
;
201 /* The bottom level has pointers to PageDesc, and is indexed by
202 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
204 #define V_L1_MIN_BITS 4
205 #define V_L1_MAX_BITS (V_L2_BITS + 3)
206 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
208 static void *l1_map
[V_L1_MAX_SIZE
];
210 /* code generation context */
211 TCGContext tcg_init_ctx
;
212 __thread TCGContext
*tcg_ctx
;
216 static void page_table_config_init(void)
220 assert(TARGET_PAGE_BITS
);
221 /* The bits remaining after N lower levels of page tables. */
222 v_l1_bits
= (L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
) % V_L2_BITS
;
223 if (v_l1_bits
< V_L1_MIN_BITS
) {
224 v_l1_bits
+= V_L2_BITS
;
227 v_l1_size
= 1 << v_l1_bits
;
228 v_l1_shift
= L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
- v_l1_bits
;
229 v_l2_levels
= v_l1_shift
/ V_L2_BITS
- 1;
231 assert(v_l1_bits
<= V_L1_MAX_BITS
);
232 assert(v_l1_shift
% V_L2_BITS
== 0);
233 assert(v_l2_levels
>= 0);
236 void cpu_gen_init(void)
238 tcg_context_init(&tcg_init_ctx
);
241 /* Encode VAL as a signed leb128 sequence at P.
242 Return P incremented past the encoded value. */
243 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
250 more
= !((val
== 0 && (byte
& 0x40) == 0)
251 || (val
== -1 && (byte
& 0x40) != 0));
261 /* Decode a signed leb128 sequence at *PP; increment *PP past the
262 decoded value. Return the decoded value. */
263 static target_long
decode_sleb128(uint8_t **pp
)
271 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
273 } while (byte
& 0x80);
274 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
275 val
|= -(target_ulong
)1 << shift
;
282 /* Encode the data collected about the instructions while compiling TB.
283 Place the data at BLOCK, and return the number of bytes consumed.
285 The logical table consists of TARGET_INSN_START_WORDS target_ulong's,
286 which come from the target's insn_start data, followed by a uintptr_t
287 which comes from the host pc of the end of the code implementing the insn.
289 Each line of the table is encoded as sleb128 deltas from the previous
290 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
291 That is, the first column is seeded with the guest pc, the last column
292 with the host pc, and the middle columns with zeros. */
294 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
296 uint8_t *highwater
= tcg_ctx
->code_gen_highwater
;
300 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
303 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
305 prev
= (j
== 0 ? tb
->pc
: 0);
307 prev
= tcg_ctx
->gen_insn_data
[i
- 1][j
];
309 p
= encode_sleb128(p
, tcg_ctx
->gen_insn_data
[i
][j
] - prev
);
311 prev
= (i
== 0 ? 0 : tcg_ctx
->gen_insn_end_off
[i
- 1]);
312 p
= encode_sleb128(p
, tcg_ctx
->gen_insn_end_off
[i
] - prev
);
314 /* Test for (pending) buffer overflow. The assumption is that any
315 one row beginning below the high water mark cannot overrun
316 the buffer completely. Thus we can test for overflow after
317 encoding a row without having to check during encoding. */
318 if (unlikely(p
> highwater
)) {
326 /* The cpu state corresponding to 'searched_pc' is restored.
327 * When reset_icount is true, current TB will be interrupted and
328 * icount should be recalculated.
330 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
331 uintptr_t searched_pc
, bool reset_icount
)
333 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
334 uintptr_t host_pc
= (uintptr_t)tb
->tc
.ptr
;
335 CPUArchState
*env
= cpu
->env_ptr
;
336 uint8_t *p
= tb
->tc
.ptr
+ tb
->tc
.size
;
337 int i
, j
, num_insns
= tb
->icount
;
338 #ifdef CONFIG_PROFILER
339 TCGProfile
*prof
= &tcg_ctx
->prof
;
340 int64_t ti
= profile_getclock();
343 searched_pc
-= GETPC_ADJ
;
345 if (searched_pc
< host_pc
) {
349 /* Reconstruct the stored insn data while looking for the point at
350 which the end of the insn exceeds the searched_pc. */
351 for (i
= 0; i
< num_insns
; ++i
) {
352 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
353 data
[j
] += decode_sleb128(&p
);
355 host_pc
+= decode_sleb128(&p
);
356 if (host_pc
> searched_pc
) {
363 if (reset_icount
&& (tb_cflags(tb
) & CF_USE_ICOUNT
)) {
365 /* Reset the cycle counter to the start of the block
366 and shift if to the number of actually executed instructions */
367 cpu
->icount_decr
.u16
.low
+= num_insns
- i
;
369 restore_state_to_opc(env
, tb
, data
);
371 #ifdef CONFIG_PROFILER
372 atomic_set(&prof
->restore_time
,
373 prof
->restore_time
+ profile_getclock() - ti
);
374 atomic_set(&prof
->restore_count
, prof
->restore_count
+ 1);
379 bool cpu_restore_state(CPUState
*cpu
, uintptr_t host_pc
, bool will_exit
)
381 TranslationBlock
*tb
;
383 uintptr_t check_offset
;
385 /* The host_pc has to be in the region of current code buffer. If
386 * it is not we will not be able to resolve it here. The two cases
387 * where host_pc will not be correct are:
389 * - fault during translation (instruction fetch)
390 * - fault from helper (not using GETPC() macro)
392 * Either way we need return early as we can't resolve it here.
394 * We are using unsigned arithmetic so if host_pc <
395 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way
396 * above the code_gen_buffer_size
398 check_offset
= host_pc
- (uintptr_t) tcg_init_ctx
.code_gen_buffer
;
400 if (check_offset
< tcg_init_ctx
.code_gen_buffer_size
) {
401 tb
= tcg_tb_lookup(host_pc
);
403 cpu_restore_state_from_tb(cpu
, tb
, host_pc
, will_exit
);
404 if (tb_cflags(tb
) & CF_NOCACHE
) {
405 /* one-shot translation, invalidate it immediately */
406 tb_phys_invalidate(tb
, -1);
416 static void page_init(void)
419 page_table_config_init();
421 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
423 #ifdef HAVE_KINFO_GETVMMAP
424 struct kinfo_vmentry
*freep
;
427 freep
= kinfo_getvmmap(getpid(), &cnt
);
430 for (i
= 0; i
< cnt
; i
++) {
431 unsigned long startaddr
, endaddr
;
433 startaddr
= freep
[i
].kve_start
;
434 endaddr
= freep
[i
].kve_end
;
435 if (h2g_valid(startaddr
)) {
436 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
438 if (h2g_valid(endaddr
)) {
439 endaddr
= h2g(endaddr
);
440 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
442 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
444 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
455 last_brk
= (unsigned long)sbrk(0);
457 f
= fopen("/compat/linux/proc/self/maps", "r");
462 unsigned long startaddr
, endaddr
;
465 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
467 if (n
== 2 && h2g_valid(startaddr
)) {
468 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
470 if (h2g_valid(endaddr
)) {
471 endaddr
= h2g(endaddr
);
475 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
487 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
493 /* Level 1. Always allocated. */
494 lp
= l1_map
+ ((index
>> v_l1_shift
) & (v_l1_size
- 1));
497 for (i
= v_l2_levels
; i
> 0; i
--) {
498 void **p
= atomic_rcu_read(lp
);
506 p
= g_new0(void *, V_L2_SIZE
);
507 existing
= atomic_cmpxchg(lp
, NULL
, p
);
508 if (unlikely(existing
)) {
514 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
517 pd
= atomic_rcu_read(lp
);
524 pd
= g_new0(PageDesc
, V_L2_SIZE
);
525 #ifndef CONFIG_USER_ONLY
529 for (i
= 0; i
< V_L2_SIZE
; i
++) {
530 qemu_spin_init(&pd
[i
].lock
);
534 existing
= atomic_cmpxchg(lp
, NULL
, pd
);
535 if (unlikely(existing
)) {
541 return pd
+ (index
& (V_L2_SIZE
- 1));
544 static inline PageDesc
*page_find(tb_page_addr_t index
)
546 return page_find_alloc(index
, 0);
549 static void page_lock_pair(PageDesc
**ret_p1
, tb_page_addr_t phys1
,
550 PageDesc
**ret_p2
, tb_page_addr_t phys2
, int alloc
);
552 /* In user-mode page locks aren't used; mmap_lock is enough */
553 #ifdef CONFIG_USER_ONLY
555 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
557 static inline void page_lock(PageDesc
*pd
)
560 static inline void page_unlock(PageDesc
*pd
)
563 static inline void page_lock_tb(const TranslationBlock
*tb
)
566 static inline void page_unlock_tb(const TranslationBlock
*tb
)
569 struct page_collection
*
570 page_collection_lock(tb_page_addr_t start
, tb_page_addr_t end
)
575 void page_collection_unlock(struct page_collection
*set
)
577 #else /* !CONFIG_USER_ONLY */
579 #ifdef CONFIG_DEBUG_TCG
581 static __thread GHashTable
*ht_pages_locked_debug
;
583 static void ht_pages_locked_debug_init(void)
585 if (ht_pages_locked_debug
) {
588 ht_pages_locked_debug
= g_hash_table_new(NULL
, NULL
);
591 static bool page_is_locked(const PageDesc
*pd
)
595 ht_pages_locked_debug_init();
596 found
= g_hash_table_lookup(ht_pages_locked_debug
, pd
);
600 static void page_lock__debug(PageDesc
*pd
)
602 ht_pages_locked_debug_init();
603 g_assert(!page_is_locked(pd
));
604 g_hash_table_insert(ht_pages_locked_debug
, pd
, pd
);
607 static void page_unlock__debug(const PageDesc
*pd
)
611 ht_pages_locked_debug_init();
612 g_assert(page_is_locked(pd
));
613 removed
= g_hash_table_remove(ht_pages_locked_debug
, pd
);
618 do_assert_page_locked(const PageDesc
*pd
, const char *file
, int line
)
620 if (unlikely(!page_is_locked(pd
))) {
621 error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
627 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
629 void assert_no_pages_locked(void)
631 ht_pages_locked_debug_init();
632 g_assert(g_hash_table_size(ht_pages_locked_debug
) == 0);
635 #else /* !CONFIG_DEBUG_TCG */
637 #define assert_page_locked(pd)
639 static inline void page_lock__debug(const PageDesc
*pd
)
643 static inline void page_unlock__debug(const PageDesc
*pd
)
647 #endif /* CONFIG_DEBUG_TCG */
649 static inline void page_lock(PageDesc
*pd
)
651 page_lock__debug(pd
);
652 qemu_spin_lock(&pd
->lock
);
655 static inline void page_unlock(PageDesc
*pd
)
657 qemu_spin_unlock(&pd
->lock
);
658 page_unlock__debug(pd
);
661 /* lock the page(s) of a TB in the correct acquisition order */
662 static inline void page_lock_tb(const TranslationBlock
*tb
)
664 page_lock_pair(NULL
, tb
->page_addr
[0], NULL
, tb
->page_addr
[1], 0);
667 static inline void page_unlock_tb(const TranslationBlock
*tb
)
669 PageDesc
*p1
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
672 if (unlikely(tb
->page_addr
[1] != -1)) {
673 PageDesc
*p2
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
681 static inline struct page_entry
*
682 page_entry_new(PageDesc
*pd
, tb_page_addr_t index
)
684 struct page_entry
*pe
= g_malloc(sizeof(*pe
));
692 static void page_entry_destroy(gpointer p
)
694 struct page_entry
*pe
= p
;
696 g_assert(pe
->locked
);
701 /* returns false on success */
702 static bool page_entry_trylock(struct page_entry
*pe
)
706 busy
= qemu_spin_trylock(&pe
->pd
->lock
);
708 g_assert(!pe
->locked
);
710 page_lock__debug(pe
->pd
);
715 static void do_page_entry_lock(struct page_entry
*pe
)
718 g_assert(!pe
->locked
);
722 static gboolean
page_entry_lock(gpointer key
, gpointer value
, gpointer data
)
724 struct page_entry
*pe
= value
;
726 do_page_entry_lock(pe
);
730 static gboolean
page_entry_unlock(gpointer key
, gpointer value
, gpointer data
)
732 struct page_entry
*pe
= value
;
742 * Trylock a page, and if successful, add the page to a collection.
743 * Returns true ("busy") if the page could not be locked; false otherwise.
745 static bool page_trylock_add(struct page_collection
*set
, tb_page_addr_t addr
)
747 tb_page_addr_t index
= addr
>> TARGET_PAGE_BITS
;
748 struct page_entry
*pe
;
751 pe
= g_tree_lookup(set
->tree
, &index
);
756 pd
= page_find(index
);
761 pe
= page_entry_new(pd
, index
);
762 g_tree_insert(set
->tree
, &pe
->index
, pe
);
765 * If this is either (1) the first insertion or (2) a page whose index
766 * is higher than any other so far, just lock the page and move on.
768 if (set
->max
== NULL
|| pe
->index
> set
->max
->index
) {
770 do_page_entry_lock(pe
);
774 * Try to acquire out-of-order lock; if busy, return busy so that we acquire
777 return page_entry_trylock(pe
);
780 static gint
tb_page_addr_cmp(gconstpointer ap
, gconstpointer bp
, gpointer udata
)
782 tb_page_addr_t a
= *(const tb_page_addr_t
*)ap
;
783 tb_page_addr_t b
= *(const tb_page_addr_t
*)bp
;
794 * Lock a range of pages ([@start,@end[) as well as the pages of all
796 * Locking order: acquire locks in ascending order of page index.
798 struct page_collection
*
799 page_collection_lock(tb_page_addr_t start
, tb_page_addr_t end
)
801 struct page_collection
*set
= g_malloc(sizeof(*set
));
802 tb_page_addr_t index
;
805 start
>>= TARGET_PAGE_BITS
;
806 end
>>= TARGET_PAGE_BITS
;
807 g_assert(start
<= end
);
809 set
->tree
= g_tree_new_full(tb_page_addr_cmp
, NULL
, NULL
,
812 assert_no_pages_locked();
815 g_tree_foreach(set
->tree
, page_entry_lock
, NULL
);
817 for (index
= start
; index
<= end
; index
++) {
818 TranslationBlock
*tb
;
821 pd
= page_find(index
);
825 if (page_trylock_add(set
, index
<< TARGET_PAGE_BITS
)) {
826 g_tree_foreach(set
->tree
, page_entry_unlock
, NULL
);
829 assert_page_locked(pd
);
830 PAGE_FOR_EACH_TB(pd
, tb
, n
) {
831 if (page_trylock_add(set
, tb
->page_addr
[0]) ||
832 (tb
->page_addr
[1] != -1 &&
833 page_trylock_add(set
, tb
->page_addr
[1]))) {
834 /* drop all locks, and reacquire in order */
835 g_tree_foreach(set
->tree
, page_entry_unlock
, NULL
);
843 void page_collection_unlock(struct page_collection
*set
)
845 /* entries are unlocked and freed via page_entry_destroy */
846 g_tree_destroy(set
->tree
);
850 #endif /* !CONFIG_USER_ONLY */
852 static void page_lock_pair(PageDesc
**ret_p1
, tb_page_addr_t phys1
,
853 PageDesc
**ret_p2
, tb_page_addr_t phys2
, int alloc
)
856 tb_page_addr_t page1
;
857 tb_page_addr_t page2
;
859 assert_memory_lock();
860 g_assert(phys1
!= -1);
862 page1
= phys1
>> TARGET_PAGE_BITS
;
863 page2
= phys2
>> TARGET_PAGE_BITS
;
865 p1
= page_find_alloc(page1
, alloc
);
869 if (likely(phys2
== -1)) {
872 } else if (page1
== page2
) {
879 p2
= page_find_alloc(page2
, alloc
);
892 #if defined(CONFIG_USER_ONLY)
893 /* Currently it is not recommended to allocate big chunks of data in
894 user mode. It will change when a dedicated libc will be used. */
895 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
896 region in which the guest needs to run. Revisit this. */
897 #define USE_STATIC_CODE_GEN_BUFFER
900 /* Minimum size of the code gen buffer. This number is randomly chosen,
901 but not so small that we can't have a fair number of TB's live. */
902 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
904 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
905 indicated, this is constrained by the range of direct branches on the
906 host cpu, as used by the TCG implementation of goto_tb. */
907 #if defined(__x86_64__)
908 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
909 #elif defined(__sparc__)
910 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
911 #elif defined(__powerpc64__)
912 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
913 #elif defined(__powerpc__)
914 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
915 #elif defined(__aarch64__)
916 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
917 #elif defined(__s390x__)
918 /* We have a +- 4GB range on the branches; leave some slop. */
919 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
920 #elif defined(__mips__)
921 /* We have a 256MB branch region, but leave room to make sure the
922 main executable is also within that region. */
923 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
925 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
928 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
930 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
931 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
932 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
934 static inline size_t size_code_gen_buffer(size_t tb_size
)
936 /* Size the buffer. */
938 #ifdef USE_STATIC_CODE_GEN_BUFFER
939 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
941 /* ??? Needs adjustments. */
942 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
943 static buffer, we could size this on RESERVED_VA, on the text
944 segment size of the executable, or continue to use the default. */
945 tb_size
= (unsigned long)(ram_size
/ 4);
948 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
949 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
951 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
952 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
958 /* In order to use J and JAL within the code_gen_buffer, we require
959 that the buffer not cross a 256MB boundary. */
960 static inline bool cross_256mb(void *addr
, size_t size
)
962 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
965 /* We weren't able to allocate a buffer without crossing that boundary,
966 so make do with the larger portion of the buffer that doesn't cross.
967 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
968 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
970 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
971 size_t size2
= buf1
+ size1
- buf2
;
979 tcg_ctx
->code_gen_buffer_size
= size1
;
984 #ifdef USE_STATIC_CODE_GEN_BUFFER
985 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
986 __attribute__((aligned(CODE_GEN_ALIGN
)));
988 static inline void *alloc_code_gen_buffer(void)
990 void *buf
= static_code_gen_buffer
;
991 void *end
= static_code_gen_buffer
+ sizeof(static_code_gen_buffer
);
994 /* page-align the beginning and end of the buffer */
995 buf
= QEMU_ALIGN_PTR_UP(buf
, qemu_real_host_page_size
);
996 end
= QEMU_ALIGN_PTR_DOWN(end
, qemu_real_host_page_size
);
1000 /* Honor a command-line option limiting the size of the buffer. */
1001 if (size
> tcg_ctx
->code_gen_buffer_size
) {
1002 size
= QEMU_ALIGN_DOWN(tcg_ctx
->code_gen_buffer_size
,
1003 qemu_real_host_page_size
);
1005 tcg_ctx
->code_gen_buffer_size
= size
;
1008 if (cross_256mb(buf
, size
)) {
1009 buf
= split_cross_256mb(buf
, size
);
1010 size
= tcg_ctx
->code_gen_buffer_size
;
1014 if (qemu_mprotect_rwx(buf
, size
)) {
1017 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
1021 #elif defined(_WIN32)
1022 static inline void *alloc_code_gen_buffer(void)
1024 size_t size
= tcg_ctx
->code_gen_buffer_size
;
1025 return VirtualAlloc(NULL
, size
, MEM_RESERVE
| MEM_COMMIT
,
1026 PAGE_EXECUTE_READWRITE
);
1029 static inline void *alloc_code_gen_buffer(void)
1031 int prot
= PROT_WRITE
| PROT_READ
| PROT_EXEC
;
1032 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
1033 uintptr_t start
= 0;
1034 size_t size
= tcg_ctx
->code_gen_buffer_size
;
1037 /* Constrain the position of the buffer based on the host cpu.
1038 Note that these addresses are chosen in concert with the
1039 addresses assigned in the relevant linker script file. */
1040 # if defined(__PIE__) || defined(__PIC__)
1041 /* Don't bother setting a preferred location if we're building
1042 a position-independent executable. We're more likely to get
1043 an address near the main executable if we let the kernel
1044 choose the address. */
1045 # elif defined(__x86_64__) && defined(MAP_32BIT)
1046 /* Force the memory down into low memory with the executable.
1047 Leave the choice of exact location with the kernel. */
1049 /* Cannot expect to map more than 800MB in low memory. */
1050 if (size
> 800u * 1024 * 1024) {
1051 tcg_ctx
->code_gen_buffer_size
= size
= 800u * 1024 * 1024;
1053 # elif defined(__sparc__)
1054 start
= 0x40000000ul
;
1055 # elif defined(__s390x__)
1056 start
= 0x90000000ul
;
1057 # elif defined(__mips__)
1058 # if _MIPS_SIM == _ABI64
1059 start
= 0x128000000ul
;
1061 start
= 0x08000000ul
;
1065 buf
= mmap((void *)start
, size
, prot
, flags
, -1, 0);
1066 if (buf
== MAP_FAILED
) {
1071 if (cross_256mb(buf
, size
)) {
1072 /* Try again, with the original still mapped, to avoid re-acquiring
1073 that 256mb crossing. This time don't specify an address. */
1075 void *buf2
= mmap(NULL
, size
, prot
, flags
, -1, 0);
1076 switch ((int)(buf2
!= MAP_FAILED
)) {
1078 if (!cross_256mb(buf2
, size
)) {
1079 /* Success! Use the new buffer. */
1083 /* Failure. Work with what we had. */
1087 /* Split the original buffer. Free the smaller half. */
1088 buf2
= split_cross_256mb(buf
, size
);
1089 size2
= tcg_ctx
->code_gen_buffer_size
;
1091 munmap(buf
+ size2
, size
- size2
);
1093 munmap(buf
, size
- size2
);
1102 /* Request large pages for the buffer. */
1103 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
1107 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
1109 static inline void code_gen_alloc(size_t tb_size
)
1111 tcg_ctx
->code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
1112 tcg_ctx
->code_gen_buffer
= alloc_code_gen_buffer();
1113 if (tcg_ctx
->code_gen_buffer
== NULL
) {
1114 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
1119 static bool tb_cmp(const void *ap
, const void *bp
)
1121 const TranslationBlock
*a
= ap
;
1122 const TranslationBlock
*b
= bp
;
1124 return a
->pc
== b
->pc
&&
1125 a
->cs_base
== b
->cs_base
&&
1126 a
->flags
== b
->flags
&&
1127 (tb_cflags(a
) & CF_HASH_MASK
) == (tb_cflags(b
) & CF_HASH_MASK
) &&
1128 a
->trace_vcpu_dstate
== b
->trace_vcpu_dstate
&&
1129 a
->page_addr
[0] == b
->page_addr
[0] &&
1130 a
->page_addr
[1] == b
->page_addr
[1];
1133 static void tb_htable_init(void)
1135 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
1137 qht_init(&tb_ctx
.htable
, tb_cmp
, CODE_GEN_HTABLE_SIZE
, mode
);
1140 /* Must be called before using the QEMU cpus. 'tb_size' is the size
1141 (in bytes) allocated to the translation buffer. Zero means default
1143 void tcg_exec_init(unsigned long tb_size
)
1149 code_gen_alloc(tb_size
);
1150 #if defined(CONFIG_SOFTMMU)
1151 /* There's no guest base to take into account, so go ahead and
1152 initialize the prologue now. */
1153 tcg_prologue_init(tcg_ctx
);
1158 * Allocate a new translation block. Flush the translation buffer if
1159 * too many translation blocks or too much generated code.
1161 static TranslationBlock
*tb_alloc(target_ulong pc
)
1163 TranslationBlock
*tb
;
1165 assert_memory_lock();
1167 tb
= tcg_tb_alloc(tcg_ctx
);
1168 if (unlikely(tb
== NULL
)) {
1174 /* call with @p->lock held */
1175 static inline void invalidate_page_bitmap(PageDesc
*p
)
1177 assert_page_locked(p
);
1178 #ifdef CONFIG_SOFTMMU
1179 g_free(p
->code_bitmap
);
1180 p
->code_bitmap
= NULL
;
1181 p
->code_write_count
= 0;
1185 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
1186 static void page_flush_tb_1(int level
, void **lp
)
1196 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1198 pd
[i
].first_tb
= (uintptr_t)NULL
;
1199 invalidate_page_bitmap(pd
+ i
);
1200 page_unlock(&pd
[i
]);
1205 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1206 page_flush_tb_1(level
- 1, pp
+ i
);
1211 static void page_flush_tb(void)
1213 int i
, l1_sz
= v_l1_size
;
1215 for (i
= 0; i
< l1_sz
; i
++) {
1216 page_flush_tb_1(v_l2_levels
, l1_map
+ i
);
1220 static gboolean
tb_host_size_iter(gpointer key
, gpointer value
, gpointer data
)
1222 const TranslationBlock
*tb
= value
;
1223 size_t *size
= data
;
1225 *size
+= tb
->tc
.size
;
1229 /* flush all the translation blocks */
1230 static void do_tb_flush(CPUState
*cpu
, run_on_cpu_data tb_flush_count
)
1233 /* If it is already been done on request of another CPU,
1236 if (tb_ctx
.tb_flush_count
!= tb_flush_count
.host_int
) {
1240 if (DEBUG_TB_FLUSH_GATE
) {
1241 size_t nb_tbs
= tcg_nb_tbs();
1242 size_t host_size
= 0;
1244 tcg_tb_foreach(tb_host_size_iter
, &host_size
);
1245 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
1246 tcg_code_size(), nb_tbs
, nb_tbs
> 0 ? host_size
/ nb_tbs
: 0);
1250 cpu_tb_jmp_cache_clear(cpu
);
1253 qht_reset_size(&tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
1256 tcg_region_reset_all();
1257 /* XXX: flush processor icache at this point if cache flush is
1259 atomic_mb_set(&tb_ctx
.tb_flush_count
, tb_ctx
.tb_flush_count
+ 1);
1265 void tb_flush(CPUState
*cpu
)
1267 if (tcg_enabled()) {
1268 unsigned tb_flush_count
= atomic_mb_read(&tb_ctx
.tb_flush_count
);
1269 async_safe_run_on_cpu(cpu
, do_tb_flush
,
1270 RUN_ON_CPU_HOST_INT(tb_flush_count
));
1275 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
1276 * so in order to prevent bit rot we compile them unconditionally in user-mode,
1277 * and let the optimizer get rid of them by wrapping their user-only callers
1278 * with if (DEBUG_TB_CHECK_GATE).
1280 #ifdef CONFIG_USER_ONLY
1282 static void do_tb_invalidate_check(void *p
, uint32_t hash
, void *userp
)
1284 TranslationBlock
*tb
= p
;
1285 target_ulong addr
= *(target_ulong
*)userp
;
1287 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
1288 printf("ERROR invalidate: address=" TARGET_FMT_lx
1289 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
1293 /* verify that all the pages have correct rights for code
1295 * Called with mmap_lock held.
1297 static void tb_invalidate_check(target_ulong address
)
1299 address
&= TARGET_PAGE_MASK
;
1300 qht_iter(&tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
1303 static void do_tb_page_check(void *p
, uint32_t hash
, void *userp
)
1305 TranslationBlock
*tb
= p
;
1308 flags1
= page_get_flags(tb
->pc
);
1309 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
1310 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
1311 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1312 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
1316 /* verify that all the pages have correct rights for code */
1317 static void tb_page_check(void)
1319 qht_iter(&tb_ctx
.htable
, do_tb_page_check
, NULL
);
1322 #endif /* CONFIG_USER_ONLY */
1325 * user-mode: call with mmap_lock held
1326 * !user-mode: call with @pd->lock held
1328 static inline void tb_page_remove(PageDesc
*pd
, TranslationBlock
*tb
)
1330 TranslationBlock
*tb1
;
1334 assert_page_locked(pd
);
1335 pprev
= &pd
->first_tb
;
1336 PAGE_FOR_EACH_TB(pd
, tb1
, n1
) {
1338 *pprev
= tb1
->page_next
[n1
];
1341 pprev
= &tb1
->page_next
[n1
];
1343 g_assert_not_reached();
1346 /* remove @orig from its @n_orig-th jump list */
1347 static inline void tb_remove_from_jmp_list(TranslationBlock
*orig
, int n_orig
)
1349 uintptr_t ptr
, ptr_locked
;
1350 TranslationBlock
*dest
;
1351 TranslationBlock
*tb
;
1355 /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
1356 ptr
= atomic_or_fetch(&orig
->jmp_dest
[n_orig
], 1);
1357 dest
= (TranslationBlock
*)(ptr
& ~1);
1362 qemu_spin_lock(&dest
->jmp_lock
);
1364 * While acquiring the lock, the jump might have been removed if the
1365 * destination TB was invalidated; check again.
1367 ptr_locked
= atomic_read(&orig
->jmp_dest
[n_orig
]);
1368 if (ptr_locked
!= ptr
) {
1369 qemu_spin_unlock(&dest
->jmp_lock
);
1371 * The only possibility is that the jump was unlinked via
1372 * tb_jump_unlink(dest). Seeing here another destination would be a bug,
1373 * because we set the LSB above.
1375 g_assert(ptr_locked
== 1 && dest
->cflags
& CF_INVALID
);
1379 * We first acquired the lock, and since the destination pointer matches,
1380 * we know for sure that @orig is in the jmp list.
1382 pprev
= &dest
->jmp_list_head
;
1383 TB_FOR_EACH_JMP(dest
, tb
, n
) {
1384 if (tb
== orig
&& n
== n_orig
) {
1385 *pprev
= tb
->jmp_list_next
[n
];
1386 /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
1387 qemu_spin_unlock(&dest
->jmp_lock
);
1390 pprev
= &tb
->jmp_list_next
[n
];
1392 g_assert_not_reached();
1395 /* reset the jump entry 'n' of a TB so that it is not chained to
1397 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
1399 uintptr_t addr
= (uintptr_t)(tb
->tc
.ptr
+ tb
->jmp_reset_offset
[n
]);
1400 tb_set_jmp_target(tb
, n
, addr
);
1403 /* remove any jumps to the TB */
1404 static inline void tb_jmp_unlink(TranslationBlock
*dest
)
1406 TranslationBlock
*tb
;
1409 qemu_spin_lock(&dest
->jmp_lock
);
1411 TB_FOR_EACH_JMP(dest
, tb
, n
) {
1412 tb_reset_jump(tb
, n
);
1413 atomic_and(&tb
->jmp_dest
[n
], (uintptr_t)NULL
| 1);
1414 /* No need to clear the list entry; setting the dest ptr is enough */
1416 dest
->jmp_list_head
= (uintptr_t)NULL
;
1418 qemu_spin_unlock(&dest
->jmp_lock
);
1422 * In user-mode, call with mmap_lock held.
1423 * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
1426 static void do_tb_phys_invalidate(TranslationBlock
*tb
, bool rm_from_page_list
)
1431 tb_page_addr_t phys_pc
;
1433 assert_memory_lock();
1435 /* make sure no further incoming jumps will be chained to this TB */
1436 qemu_spin_lock(&tb
->jmp_lock
);
1437 atomic_set(&tb
->cflags
, tb
->cflags
| CF_INVALID
);
1438 qemu_spin_unlock(&tb
->jmp_lock
);
1440 /* remove the TB from the hash list */
1441 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1442 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb_cflags(tb
) & CF_HASH_MASK
,
1443 tb
->trace_vcpu_dstate
);
1444 if (!(tb
->cflags
& CF_NOCACHE
) &&
1445 !qht_remove(&tb_ctx
.htable
, tb
, h
)) {
1449 /* remove the TB from the page list */
1450 if (rm_from_page_list
) {
1451 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1452 tb_page_remove(p
, tb
);
1453 invalidate_page_bitmap(p
);
1454 if (tb
->page_addr
[1] != -1) {
1455 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1456 tb_page_remove(p
, tb
);
1457 invalidate_page_bitmap(p
);
1461 /* remove the TB from the hash list */
1462 h
= tb_jmp_cache_hash_func(tb
->pc
);
1464 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1465 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1469 /* suppress this TB from the two jump lists */
1470 tb_remove_from_jmp_list(tb
, 0);
1471 tb_remove_from_jmp_list(tb
, 1);
1473 /* suppress any remaining jumps to this TB */
1476 atomic_set(&tcg_ctx
->tb_phys_invalidate_count
,
1477 tcg_ctx
->tb_phys_invalidate_count
+ 1);
1480 static void tb_phys_invalidate__locked(TranslationBlock
*tb
)
1482 do_tb_phys_invalidate(tb
, true);
1485 /* invalidate one TB
1487 * Called with mmap_lock held in user-mode.
1489 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
1491 if (page_addr
== -1 && tb
->page_addr
[0] != -1) {
1493 do_tb_phys_invalidate(tb
, true);
1496 do_tb_phys_invalidate(tb
, false);
1500 #ifdef CONFIG_SOFTMMU
1501 /* call with @p->lock held */
1502 static void build_page_bitmap(PageDesc
*p
)
1504 int n
, tb_start
, tb_end
;
1505 TranslationBlock
*tb
;
1507 assert_page_locked(p
);
1508 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1510 PAGE_FOR_EACH_TB(p
, tb
, n
) {
1511 /* NOTE: this is subtle as a TB may span two physical pages */
1513 /* NOTE: tb_end may be after the end of the page, but
1514 it is not a problem */
1515 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1516 tb_end
= tb_start
+ tb
->size
;
1517 if (tb_end
> TARGET_PAGE_SIZE
) {
1518 tb_end
= TARGET_PAGE_SIZE
;
1522 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1524 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1529 /* add the tb in the target page and protect it if necessary
1531 * Called with mmap_lock held for user-mode emulation.
1532 * Called with @p->lock held in !user-mode.
1534 static inline void tb_page_add(PageDesc
*p
, TranslationBlock
*tb
,
1535 unsigned int n
, tb_page_addr_t page_addr
)
1537 #ifndef CONFIG_USER_ONLY
1538 bool page_already_protected
;
1541 assert_page_locked(p
);
1543 tb
->page_addr
[n
] = page_addr
;
1544 tb
->page_next
[n
] = p
->first_tb
;
1545 #ifndef CONFIG_USER_ONLY
1546 page_already_protected
= p
->first_tb
!= (uintptr_t)NULL
;
1548 p
->first_tb
= (uintptr_t)tb
| n
;
1549 invalidate_page_bitmap(p
);
1551 #if defined(CONFIG_USER_ONLY)
1552 if (p
->flags
& PAGE_WRITE
) {
1557 /* force the host page as non writable (writes will have a
1558 page fault + mprotect overhead) */
1559 page_addr
&= qemu_host_page_mask
;
1561 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1562 addr
+= TARGET_PAGE_SIZE
) {
1564 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1569 p2
->flags
&= ~PAGE_WRITE
;
1571 mprotect(g2h(page_addr
), qemu_host_page_size
,
1572 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1573 if (DEBUG_TB_INVALIDATE_GATE
) {
1574 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT
"\n", page_addr
);
1578 /* if some code is already present, then the pages are already
1579 protected. So we handle the case where only the first TB is
1580 allocated in a physical page */
1581 if (!page_already_protected
) {
1582 tlb_protect_code(page_addr
);
1587 /* add a new TB and link it to the physical page tables. phys_page2 is
1588 * (-1) to indicate that only one page contains the TB.
1590 * Called with mmap_lock held for user-mode emulation.
1592 * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
1593 * Note that in !user-mode, another thread might have already added a TB
1594 * for the same block of guest code that @tb corresponds to. In that case,
1595 * the caller should discard the original @tb, and use instead the returned TB.
1597 static TranslationBlock
*
1598 tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1599 tb_page_addr_t phys_page2
)
1602 PageDesc
*p2
= NULL
;
1604 assert_memory_lock();
1606 if (phys_pc
== -1) {
1608 * If the TB is not associated with a physical RAM page then
1609 * it must be a temporary one-insn TB, and we have nothing to do
1610 * except fill in the page_addr[] fields.
1612 assert(tb
->cflags
& CF_NOCACHE
);
1613 tb
->page_addr
[0] = tb
->page_addr
[1] = -1;
1618 * Add the TB to the page list, acquiring first the pages's locks.
1619 * We keep the locks held until after inserting the TB in the hash table,
1620 * so that if the insertion fails we know for sure that the TBs are still
1621 * in the page descriptors.
1622 * Note that inserting into the hash table first isn't an option, since
1623 * we can only insert TBs that are fully initialized.
1625 page_lock_pair(&p
, phys_pc
, &p2
, phys_page2
, 1);
1626 tb_page_add(p
, tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1628 tb_page_add(p2
, tb
, 1, phys_page2
);
1630 tb
->page_addr
[1] = -1;
1633 if (!(tb
->cflags
& CF_NOCACHE
)) {
1634 void *existing_tb
= NULL
;
1637 /* add in the hash table */
1638 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->cflags
& CF_HASH_MASK
,
1639 tb
->trace_vcpu_dstate
);
1640 qht_insert(&tb_ctx
.htable
, tb
, h
, &existing_tb
);
1642 /* remove TB from the page(s) if we couldn't insert it */
1643 if (unlikely(existing_tb
)) {
1644 tb_page_remove(p
, tb
);
1645 invalidate_page_bitmap(p
);
1647 tb_page_remove(p2
, tb
);
1648 invalidate_page_bitmap(p2
);
1654 if (p2
&& p2
!= p
) {
1659 #ifdef CONFIG_USER_ONLY
1660 if (DEBUG_TB_CHECK_GATE
) {
1667 /* Called with mmap_lock held for user mode emulation. */
1668 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1669 target_ulong pc
, target_ulong cs_base
,
1670 uint32_t flags
, int cflags
)
1672 CPUArchState
*env
= cpu
->env_ptr
;
1673 TranslationBlock
*tb
, *existing_tb
;
1674 tb_page_addr_t phys_pc
, phys_page2
;
1675 target_ulong virt_page2
;
1676 tcg_insn_unit
*gen_code_buf
;
1677 int gen_code_size
, search_size
, max_insns
;
1678 #ifdef CONFIG_PROFILER
1679 TCGProfile
*prof
= &tcg_ctx
->prof
;
1682 assert_memory_lock();
1684 phys_pc
= get_page_addr_code(env
, pc
);
1686 if (phys_pc
== -1) {
1687 /* Generate a temporary TB with 1 insn in it */
1688 cflags
&= ~CF_COUNT_MASK
;
1689 cflags
|= CF_NOCACHE
| 1;
1692 cflags
&= ~CF_CLUSTER_MASK
;
1693 cflags
|= cpu
->cluster_index
<< CF_CLUSTER_SHIFT
;
1695 max_insns
= cflags
& CF_COUNT_MASK
;
1696 if (max_insns
== 0) {
1697 max_insns
= CF_COUNT_MASK
;
1699 if (max_insns
> TCG_MAX_INSNS
) {
1700 max_insns
= TCG_MAX_INSNS
;
1702 if (cpu
->singlestep_enabled
|| singlestep
) {
1708 if (unlikely(!tb
)) {
1709 /* flush must be done */
1712 /* Make the execution loop process the flush as soon as possible. */
1713 cpu
->exception_index
= EXCP_INTERRUPT
;
1717 gen_code_buf
= tcg_ctx
->code_gen_ptr
;
1718 tb
->tc
.ptr
= gen_code_buf
;
1720 tb
->cs_base
= cs_base
;
1722 tb
->cflags
= cflags
;
1723 tb
->trace_vcpu_dstate
= *cpu
->trace_dstate
;
1724 tcg_ctx
->tb_cflags
= cflags
;
1727 #ifdef CONFIG_PROFILER
1728 /* includes aborted translations because of exceptions */
1729 atomic_set(&prof
->tb_count1
, prof
->tb_count1
+ 1);
1730 ti
= profile_getclock();
1733 tcg_func_start(tcg_ctx
);
1735 tcg_ctx
->cpu
= ENV_GET_CPU(env
);
1736 gen_intermediate_code(cpu
, tb
, max_insns
);
1737 tcg_ctx
->cpu
= NULL
;
1739 trace_translate_block(tb
, tb
->pc
, tb
->tc
.ptr
);
1741 /* generate machine code */
1742 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1743 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1744 tcg_ctx
->tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1745 if (TCG_TARGET_HAS_direct_jump
) {
1746 tcg_ctx
->tb_jmp_insn_offset
= tb
->jmp_target_arg
;
1747 tcg_ctx
->tb_jmp_target_addr
= NULL
;
1749 tcg_ctx
->tb_jmp_insn_offset
= NULL
;
1750 tcg_ctx
->tb_jmp_target_addr
= tb
->jmp_target_arg
;
1753 #ifdef CONFIG_PROFILER
1754 atomic_set(&prof
->tb_count
, prof
->tb_count
+ 1);
1755 atomic_set(&prof
->interm_time
, prof
->interm_time
+ profile_getclock() - ti
);
1756 ti
= profile_getclock();
1759 gen_code_size
= tcg_gen_code(tcg_ctx
, tb
);
1760 if (unlikely(gen_code_size
< 0)) {
1761 switch (gen_code_size
) {
1764 * Overflow of code_gen_buffer, or the current slice of it.
1766 * TODO: We don't need to re-do gen_intermediate_code, nor
1767 * should we re-do the tcg optimization currently hidden
1768 * inside tcg_gen_code. All that should be required is to
1769 * flush the TBs, allocate a new TB, re-initialize it per
1770 * above, and re-do the actual code generation.
1772 goto buffer_overflow
;
1776 * The code generated for the TranslationBlock is too large.
1777 * The maximum size allowed by the unwind info is 64k.
1778 * There may be stricter constraints from relocations
1779 * in the tcg backend.
1781 * Try again with half as many insns as we attempted this time.
1782 * If a single insn overflows, there's a bug somewhere...
1784 max_insns
= tb
->icount
;
1785 assert(max_insns
> 1);
1790 g_assert_not_reached();
1793 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1794 if (unlikely(search_size
< 0)) {
1795 goto buffer_overflow
;
1797 tb
->tc
.size
= gen_code_size
;
1799 #ifdef CONFIG_PROFILER
1800 atomic_set(&prof
->code_time
, prof
->code_time
+ profile_getclock() - ti
);
1801 atomic_set(&prof
->code_in_len
, prof
->code_in_len
+ tb
->size
);
1802 atomic_set(&prof
->code_out_len
, prof
->code_out_len
+ gen_code_size
);
1803 atomic_set(&prof
->search_out_len
, prof
->search_out_len
+ search_size
);
1807 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1808 qemu_log_in_addr_range(tb
->pc
)) {
1810 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1811 if (tcg_ctx
->data_gen_ptr
) {
1812 size_t code_size
= tcg_ctx
->data_gen_ptr
- tb
->tc
.ptr
;
1813 size_t data_size
= gen_code_size
- code_size
;
1816 log_disas(tb
->tc
.ptr
, code_size
);
1818 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1819 if (sizeof(tcg_target_ulong
) == 8) {
1820 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1821 (uintptr_t)tcg_ctx
->data_gen_ptr
+ i
,
1822 *(uint64_t *)(tcg_ctx
->data_gen_ptr
+ i
));
1824 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1825 (uintptr_t)tcg_ctx
->data_gen_ptr
+ i
,
1826 *(uint32_t *)(tcg_ctx
->data_gen_ptr
+ i
));
1830 log_disas(tb
->tc
.ptr
, gen_code_size
);
1838 atomic_set(&tcg_ctx
->code_gen_ptr
, (void *)
1839 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1842 /* init jump list */
1843 qemu_spin_init(&tb
->jmp_lock
);
1844 tb
->jmp_list_head
= (uintptr_t)NULL
;
1845 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1846 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1847 tb
->jmp_dest
[0] = (uintptr_t)NULL
;
1848 tb
->jmp_dest
[1] = (uintptr_t)NULL
;
1850 /* init original jump addresses which have been set during tcg_gen_code() */
1851 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1852 tb_reset_jump(tb
, 0);
1854 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1855 tb_reset_jump(tb
, 1);
1858 /* check next page if needed */
1859 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1861 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1862 phys_page2
= get_page_addr_code(env
, virt_page2
);
1865 * No explicit memory barrier is required -- tb_link_page() makes the
1866 * TB visible in a consistent state.
1868 existing_tb
= tb_link_page(tb
, phys_pc
, phys_page2
);
1869 /* if the TB already exists, discard what we just translated */
1870 if (unlikely(existing_tb
!= tb
)) {
1871 uintptr_t orig_aligned
= (uintptr_t)gen_code_buf
;
1873 orig_aligned
-= ROUND_UP(sizeof(*tb
), qemu_icache_linesize
);
1874 atomic_set(&tcg_ctx
->code_gen_ptr
, (void *)orig_aligned
);
1882 * @p must be non-NULL.
1883 * user-mode: call with mmap_lock held.
1884 * !user-mode: call with all @pages locked.
1887 tb_invalidate_phys_page_range__locked(struct page_collection
*pages
,
1888 PageDesc
*p
, tb_page_addr_t start
,
1890 int is_cpu_write_access
)
1892 TranslationBlock
*tb
;
1893 tb_page_addr_t tb_start
, tb_end
;
1895 #ifdef TARGET_HAS_PRECISE_SMC
1896 CPUState
*cpu
= current_cpu
;
1897 CPUArchState
*env
= NULL
;
1898 int current_tb_not_found
= is_cpu_write_access
;
1899 TranslationBlock
*current_tb
= NULL
;
1900 int current_tb_modified
= 0;
1901 target_ulong current_pc
= 0;
1902 target_ulong current_cs_base
= 0;
1903 uint32_t current_flags
= 0;
1904 #endif /* TARGET_HAS_PRECISE_SMC */
1906 assert_page_locked(p
);
1908 #if defined(TARGET_HAS_PRECISE_SMC)
1914 /* we remove all the TBs in the range [start, end[ */
1915 /* XXX: see if in some cases it could be faster to invalidate all
1917 PAGE_FOR_EACH_TB(p
, tb
, n
) {
1918 assert_page_locked(p
);
1919 /* NOTE: this is subtle as a TB may span two physical pages */
1921 /* NOTE: tb_end may be after the end of the page, but
1922 it is not a problem */
1923 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1924 tb_end
= tb_start
+ tb
->size
;
1926 tb_start
= tb
->page_addr
[1];
1927 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1929 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1930 #ifdef TARGET_HAS_PRECISE_SMC
1931 if (current_tb_not_found
) {
1932 current_tb_not_found
= 0;
1934 if (cpu
->mem_io_pc
) {
1935 /* now we have a real cpu fault */
1936 current_tb
= tcg_tb_lookup(cpu
->mem_io_pc
);
1939 if (current_tb
== tb
&&
1940 (tb_cflags(current_tb
) & CF_COUNT_MASK
) != 1) {
1941 /* If we are modifying the current TB, we must stop
1942 its execution. We could be more precise by checking
1943 that the modification is after the current PC, but it
1944 would require a specialized function to partially
1945 restore the CPU state */
1947 current_tb_modified
= 1;
1948 cpu_restore_state_from_tb(cpu
, current_tb
,
1949 cpu
->mem_io_pc
, true);
1950 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1953 #endif /* TARGET_HAS_PRECISE_SMC */
1954 tb_phys_invalidate__locked(tb
);
1957 #if !defined(CONFIG_USER_ONLY)
1958 /* if no code remaining, no need to continue to use slow writes */
1960 invalidate_page_bitmap(p
);
1961 tlb_unprotect_code(start
);
1964 #ifdef TARGET_HAS_PRECISE_SMC
1965 if (current_tb_modified
) {
1966 page_collection_unlock(pages
);
1967 /* Force execution of one insn next time. */
1968 cpu
->cflags_next_tb
= 1 | curr_cflags();
1970 cpu_loop_exit_noexc(cpu
);
1976 * Invalidate all TBs which intersect with the target physical address range
1977 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1978 * 'is_cpu_write_access' should be true if called from a real cpu write
1979 * access: the virtual CPU will exit the current TB if code is modified inside
1982 * Called with mmap_lock held for user-mode emulation
1984 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1985 int is_cpu_write_access
)
1987 struct page_collection
*pages
;
1990 assert_memory_lock();
1992 p
= page_find(start
>> TARGET_PAGE_BITS
);
1996 pages
= page_collection_lock(start
, end
);
1997 tb_invalidate_phys_page_range__locked(pages
, p
, start
, end
,
1998 is_cpu_write_access
);
1999 page_collection_unlock(pages
);
2003 * Invalidate all TBs which intersect with the target physical address range
2004 * [start;end[. NOTE: start and end may refer to *different* physical pages.
2005 * 'is_cpu_write_access' should be true if called from a real cpu write
2006 * access: the virtual CPU will exit the current TB if code is modified inside
2009 * Called with mmap_lock held for user-mode emulation.
2011 #ifdef CONFIG_SOFTMMU
2012 void tb_invalidate_phys_range(ram_addr_t start
, ram_addr_t end
)
2014 void tb_invalidate_phys_range(target_ulong start
, target_ulong end
)
2017 struct page_collection
*pages
;
2018 tb_page_addr_t next
;
2020 assert_memory_lock();
2022 pages
= page_collection_lock(start
, end
);
2023 for (next
= (start
& TARGET_PAGE_MASK
) + TARGET_PAGE_SIZE
;
2025 start
= next
, next
+= TARGET_PAGE_SIZE
) {
2026 PageDesc
*pd
= page_find(start
>> TARGET_PAGE_BITS
);
2027 tb_page_addr_t bound
= MIN(next
, end
);
2032 tb_invalidate_phys_page_range__locked(pages
, pd
, start
, bound
, 0);
2034 page_collection_unlock(pages
);
2037 #ifdef CONFIG_SOFTMMU
2038 /* len must be <= 8 and start must be a multiple of len.
2039 * Called via softmmu_template.h when code areas are written to with
2040 * iothread mutex not held.
2042 * Call with all @pages in the range [@start, @start + len[ locked.
2044 void tb_invalidate_phys_page_fast(struct page_collection
*pages
,
2045 tb_page_addr_t start
, int len
)
2049 assert_memory_lock();
2051 p
= page_find(start
>> TARGET_PAGE_BITS
);
2056 assert_page_locked(p
);
2057 if (!p
->code_bitmap
&&
2058 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
2059 build_page_bitmap(p
);
2061 if (p
->code_bitmap
) {
2065 nr
= start
& ~TARGET_PAGE_MASK
;
2066 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
2067 if (b
& ((1 << len
) - 1)) {
2072 tb_invalidate_phys_page_range__locked(pages
, p
, start
, start
+ len
, 1);
2076 /* Called with mmap_lock held. If pc is not 0 then it indicates the
2077 * host PC of the faulting store instruction that caused this invalidate.
2078 * Returns true if the caller needs to abort execution of the current
2079 * TB (because it was modified by this store and the guest CPU has
2080 * precise-SMC semantics).
2082 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
2084 TranslationBlock
*tb
;
2087 #ifdef TARGET_HAS_PRECISE_SMC
2088 TranslationBlock
*current_tb
= NULL
;
2089 CPUState
*cpu
= current_cpu
;
2090 CPUArchState
*env
= NULL
;
2091 int current_tb_modified
= 0;
2092 target_ulong current_pc
= 0;
2093 target_ulong current_cs_base
= 0;
2094 uint32_t current_flags
= 0;
2097 assert_memory_lock();
2099 addr
&= TARGET_PAGE_MASK
;
2100 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2105 #ifdef TARGET_HAS_PRECISE_SMC
2106 if (p
->first_tb
&& pc
!= 0) {
2107 current_tb
= tcg_tb_lookup(pc
);
2113 assert_page_locked(p
);
2114 PAGE_FOR_EACH_TB(p
, tb
, n
) {
2115 #ifdef TARGET_HAS_PRECISE_SMC
2116 if (current_tb
== tb
&&
2117 (tb_cflags(current_tb
) & CF_COUNT_MASK
) != 1) {
2118 /* If we are modifying the current TB, we must stop
2119 its execution. We could be more precise by checking
2120 that the modification is after the current PC, but it
2121 would require a specialized function to partially
2122 restore the CPU state */
2124 current_tb_modified
= 1;
2125 cpu_restore_state_from_tb(cpu
, current_tb
, pc
, true);
2126 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
2129 #endif /* TARGET_HAS_PRECISE_SMC */
2130 tb_phys_invalidate(tb
, addr
);
2132 p
->first_tb
= (uintptr_t)NULL
;
2133 #ifdef TARGET_HAS_PRECISE_SMC
2134 if (current_tb_modified
) {
2135 /* Force execution of one insn next time. */
2136 cpu
->cflags_next_tb
= 1 | curr_cflags();
2145 /* user-mode: call with mmap_lock held */
2146 void tb_check_watchpoint(CPUState
*cpu
)
2148 TranslationBlock
*tb
;
2150 assert_memory_lock();
2152 tb
= tcg_tb_lookup(cpu
->mem_io_pc
);
2154 /* We can use retranslation to find the PC. */
2155 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
, true);
2156 tb_phys_invalidate(tb
, -1);
2158 /* The exception probably happened in a helper. The CPU state should
2159 have been saved before calling it. Fetch the PC from there. */
2160 CPUArchState
*env
= cpu
->env_ptr
;
2161 target_ulong pc
, cs_base
;
2162 tb_page_addr_t addr
;
2165 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
2166 addr
= get_page_addr_code(env
, pc
);
2168 tb_invalidate_phys_range(addr
, addr
+ 1);
2173 #ifndef CONFIG_USER_ONLY
2174 /* in deterministic execution mode, instructions doing device I/Os
2175 * must be at the end of the TB.
2177 * Called by softmmu_template.h, with iothread mutex not held.
2179 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
2181 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
2182 CPUArchState
*env
= cpu
->env_ptr
;
2184 TranslationBlock
*tb
;
2187 tb
= tcg_tb_lookup(retaddr
);
2189 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
2192 cpu_restore_state_from_tb(cpu
, tb
, retaddr
, true);
2194 /* On MIPS and SH, delay slot instructions can only be restarted if
2195 they were already the first instruction in the TB. If this is not
2196 the first instruction in a TB then re-execute the preceding
2199 #if defined(TARGET_MIPS)
2200 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0
2201 && env
->active_tc
.PC
!= tb
->pc
) {
2202 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
2203 cpu
->icount_decr
.u16
.low
++;
2204 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
2207 #elif defined(TARGET_SH4)
2208 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
2209 && env
->pc
!= tb
->pc
) {
2211 cpu
->icount_decr
.u16
.low
++;
2212 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
2217 /* Generate a new TB executing the I/O insn. */
2218 cpu
->cflags_next_tb
= curr_cflags() | CF_LAST_IO
| n
;
2220 if (tb_cflags(tb
) & CF_NOCACHE
) {
2222 /* Invalidate original TB if this TB was generated in
2223 * cpu_exec_nocache() */
2224 tb_phys_invalidate(tb
->orig_tb
, -1);
2229 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
2230 * the first in the TB) then we end up generating a whole new TB and
2231 * repeating the fault, which is horribly inefficient.
2232 * Better would be to execute just this insn uncached, or generate a
2235 cpu_loop_exit_noexc(cpu
);
2238 static void tb_jmp_cache_clear_page(CPUState
*cpu
, target_ulong page_addr
)
2240 unsigned int i
, i0
= tb_jmp_cache_hash_page(page_addr
);
2242 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
2243 atomic_set(&cpu
->tb_jmp_cache
[i0
+ i
], NULL
);
2247 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
2249 /* Discard jump cache entries for any tb which might potentially
2250 overlap the flushed page. */
2251 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
2252 tb_jmp_cache_clear_page(cpu
, addr
);
2255 static void print_qht_statistics(struct qht_stats hst
)
2257 uint32_t hgram_opts
;
2261 if (!hst
.head_buckets
) {
2264 qemu_printf("TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
2265 hst
.used_head_buckets
, hst
.head_buckets
,
2266 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
2268 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
2269 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
2270 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
2271 hgram_opts
|= QDIST_PR_NODECIMAL
;
2273 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
2274 qemu_printf("TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
2275 qdist_avg(&hst
.occupancy
) * 100, hgram
);
2278 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
2279 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
2280 if (hgram_bins
> 10) {
2284 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
2286 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
2287 qemu_printf("TB hash avg chain %0.3f buckets. Histogram: %s\n",
2288 qdist_avg(&hst
.chain
), hgram
);
2292 struct tb_tree_stats
{
2296 size_t max_target_size
;
2297 size_t direct_jmp_count
;
2298 size_t direct_jmp2_count
;
2302 static gboolean
tb_tree_stats_iter(gpointer key
, gpointer value
, gpointer data
)
2304 const TranslationBlock
*tb
= value
;
2305 struct tb_tree_stats
*tst
= data
;
2308 tst
->host_size
+= tb
->tc
.size
;
2309 tst
->target_size
+= tb
->size
;
2310 if (tb
->size
> tst
->max_target_size
) {
2311 tst
->max_target_size
= tb
->size
;
2313 if (tb
->page_addr
[1] != -1) {
2316 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
2317 tst
->direct_jmp_count
++;
2318 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
2319 tst
->direct_jmp2_count
++;
2325 void dump_exec_info(void)
2327 struct tb_tree_stats tst
= {};
2328 struct qht_stats hst
;
2329 size_t nb_tbs
, flush_full
, flush_part
, flush_elide
;
2331 tcg_tb_foreach(tb_tree_stats_iter
, &tst
);
2332 nb_tbs
= tst
.nb_tbs
;
2333 /* XXX: avoid using doubles ? */
2334 qemu_printf("Translation buffer state:\n");
2336 * Report total code size including the padding and TB structs;
2337 * otherwise users might think "-tb-size" is not honoured.
2338 * For avg host size we use the precise numbers from tb_tree_stats though.
2340 qemu_printf("gen code size %zu/%zu\n",
2341 tcg_code_size(), tcg_code_capacity());
2342 qemu_printf("TB count %zu\n", nb_tbs
);
2343 qemu_printf("TB avg target size %zu max=%zu bytes\n",
2344 nb_tbs
? tst
.target_size
/ nb_tbs
: 0,
2345 tst
.max_target_size
);
2346 qemu_printf("TB avg host size %zu bytes (expansion ratio: %0.1f)\n",
2347 nb_tbs
? tst
.host_size
/ nb_tbs
: 0,
2348 tst
.target_size
? (double)tst
.host_size
/ tst
.target_size
: 0);
2349 qemu_printf("cross page TB count %zu (%zu%%)\n", tst
.cross_page
,
2350 nb_tbs
? (tst
.cross_page
* 100) / nb_tbs
: 0);
2351 qemu_printf("direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n",
2352 tst
.direct_jmp_count
,
2353 nb_tbs
? (tst
.direct_jmp_count
* 100) / nb_tbs
: 0,
2354 tst
.direct_jmp2_count
,
2355 nb_tbs
? (tst
.direct_jmp2_count
* 100) / nb_tbs
: 0);
2357 qht_statistics_init(&tb_ctx
.htable
, &hst
);
2358 print_qht_statistics(hst
);
2359 qht_statistics_destroy(&hst
);
2361 qemu_printf("\nStatistics:\n");
2362 qemu_printf("TB flush count %u\n",
2363 atomic_read(&tb_ctx
.tb_flush_count
));
2364 qemu_printf("TB invalidate count %zu\n",
2365 tcg_tb_phys_invalidate_count());
2367 tlb_flush_counts(&flush_full
, &flush_part
, &flush_elide
);
2368 qemu_printf("TLB full flushes %zu\n", flush_full
);
2369 qemu_printf("TLB partial flushes %zu\n", flush_part
);
2370 qemu_printf("TLB elided flushes %zu\n", flush_elide
);
2374 void dump_opcount_info(void)
2376 tcg_dump_op_count();
2379 #else /* CONFIG_USER_ONLY */
2381 void cpu_interrupt(CPUState
*cpu
, int mask
)
2383 g_assert(qemu_mutex_iothread_locked());
2384 cpu
->interrupt_request
|= mask
;
2385 atomic_set(&cpu
->icount_decr
.u16
.high
, -1);
2389 * Walks guest process memory "regions" one by one
2390 * and calls callback function 'fn' for each region.
2392 struct walk_memory_regions_data
{
2393 walk_memory_regions_fn fn
;
2399 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
2400 target_ulong end
, int new_prot
)
2402 if (data
->start
!= -1u) {
2403 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
2409 data
->start
= (new_prot
? end
: -1u);
2410 data
->prot
= new_prot
;
2415 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
2416 target_ulong base
, int level
, void **lp
)
2422 return walk_memory_regions_end(data
, base
, 0);
2428 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2429 int prot
= pd
[i
].flags
;
2431 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2432 if (prot
!= data
->prot
) {
2433 rc
= walk_memory_regions_end(data
, pa
, prot
);
2442 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2443 pa
= base
| ((target_ulong
)i
<<
2444 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
2445 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2455 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2457 struct walk_memory_regions_data data
;
2458 uintptr_t i
, l1_sz
= v_l1_size
;
2465 for (i
= 0; i
< l1_sz
; i
++) {
2466 target_ulong base
= i
<< (v_l1_shift
+ TARGET_PAGE_BITS
);
2467 int rc
= walk_memory_regions_1(&data
, base
, v_l2_levels
, l1_map
+ i
);
2473 return walk_memory_regions_end(&data
, 0, 0);
2476 static int dump_region(void *priv
, target_ulong start
,
2477 target_ulong end
, unsigned long prot
)
2479 FILE *f
= (FILE *)priv
;
2481 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
2482 " "TARGET_FMT_lx
" %c%c%c\n",
2483 start
, end
, end
- start
,
2484 ((prot
& PAGE_READ
) ? 'r' : '-'),
2485 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2486 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2491 /* dump memory mappings */
2492 void page_dump(FILE *f
)
2494 const int length
= sizeof(target_ulong
) * 2;
2495 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
2496 length
, "start", length
, "end", length
, "size", "prot");
2497 walk_memory_regions(f
, dump_region
);
2500 int page_get_flags(target_ulong address
)
2504 p
= page_find(address
>> TARGET_PAGE_BITS
);
2511 /* Modify the flags of a page and invalidate the code if necessary.
2512 The flag PAGE_WRITE_ORG is positioned automatically depending
2513 on PAGE_WRITE. The mmap_lock should already be held. */
2514 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2516 target_ulong addr
, len
;
2518 /* This function should never be called with addresses outside the
2519 guest address space. If this assert fires, it probably indicates
2520 a missing call to h2g_valid. */
2521 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2522 assert(end
<= ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2524 assert(start
< end
);
2525 assert_memory_lock();
2527 start
= start
& TARGET_PAGE_MASK
;
2528 end
= TARGET_PAGE_ALIGN(end
);
2530 if (flags
& PAGE_WRITE
) {
2531 flags
|= PAGE_WRITE_ORG
;
2534 for (addr
= start
, len
= end
- start
;
2536 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2537 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2539 /* If the write protection bit is set, then we invalidate
2541 if (!(p
->flags
& PAGE_WRITE
) &&
2542 (flags
& PAGE_WRITE
) &&
2544 tb_invalidate_phys_page(addr
, 0);
2550 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2556 /* This function should never be called with addresses outside the
2557 guest address space. If this assert fires, it probably indicates
2558 a missing call to h2g_valid. */
2559 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2560 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2566 if (start
+ len
- 1 < start
) {
2567 /* We've wrapped around. */
2571 /* must do before we loose bits in the next step */
2572 end
= TARGET_PAGE_ALIGN(start
+ len
);
2573 start
= start
& TARGET_PAGE_MASK
;
2575 for (addr
= start
, len
= end
- start
;
2577 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2578 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2582 if (!(p
->flags
& PAGE_VALID
)) {
2586 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
2589 if (flags
& PAGE_WRITE
) {
2590 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2593 /* unprotect the page if it was put read-only because it
2594 contains translated code */
2595 if (!(p
->flags
& PAGE_WRITE
)) {
2596 if (!page_unprotect(addr
, 0)) {
2605 /* called from signal handler: invalidate the code and unprotect the
2606 * page. Return 0 if the fault was not handled, 1 if it was handled,
2607 * and 2 if it was handled but the caller must cause the TB to be
2608 * immediately exited. (We can only return 2 if the 'pc' argument is
2611 int page_unprotect(target_ulong address
, uintptr_t pc
)
2614 bool current_tb_invalidated
;
2616 target_ulong host_start
, host_end
, addr
;
2618 /* Technically this isn't safe inside a signal handler. However we
2619 know this only ever happens in a synchronous SEGV handler, so in
2620 practice it seems to be ok. */
2623 p
= page_find(address
>> TARGET_PAGE_BITS
);
2629 /* if the page was really writable, then we change its
2630 protection back to writable */
2631 if (p
->flags
& PAGE_WRITE_ORG
) {
2632 current_tb_invalidated
= false;
2633 if (p
->flags
& PAGE_WRITE
) {
2634 /* If the page is actually marked WRITE then assume this is because
2635 * this thread raced with another one which got here first and
2636 * set the page to PAGE_WRITE and did the TB invalidate for us.
2638 #ifdef TARGET_HAS_PRECISE_SMC
2639 TranslationBlock
*current_tb
= tcg_tb_lookup(pc
);
2641 current_tb_invalidated
= tb_cflags(current_tb
) & CF_INVALID
;
2645 host_start
= address
& qemu_host_page_mask
;
2646 host_end
= host_start
+ qemu_host_page_size
;
2649 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2650 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2651 p
->flags
|= PAGE_WRITE
;
2654 /* and since the content will be modified, we must invalidate
2655 the corresponding translated code. */
2656 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2657 #ifdef CONFIG_USER_ONLY
2658 if (DEBUG_TB_CHECK_GATE
) {
2659 tb_invalidate_check(addr
);
2663 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2667 /* If current TB was invalidated return to main loop */
2668 return current_tb_invalidated
? 2 : 1;
2673 #endif /* CONFIG_USER_ONLY */
2675 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2676 void tcg_flush_softmmu_tlb(CPUState
*cs
)
2678 #ifdef CONFIG_SOFTMMU