iotests: filter few more luks specific create options
[qemu/ar7.git] / accel / tcg / translate-all.c
blob2afa46bd2b1fd9a0e3527f64cea4b52f23679335
1 /*
2 * Host code generation
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/units.h"
22 #include "qemu-common.h"
24 #define NO_CPU_IO_DEFS
25 #include "cpu.h"
26 #include "trace.h"
27 #include "disas/disas.h"
28 #include "exec/exec-all.h"
29 #include "tcg/tcg.h"
30 #if defined(CONFIG_USER_ONLY)
31 #include "qemu.h"
32 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
33 #include <sys/param.h>
34 #if __FreeBSD_version >= 700104
35 #define HAVE_KINFO_GETVMMAP
36 #define sigqueue sigqueue_freebsd /* avoid redefinition */
37 #include <sys/proc.h>
38 #include <machine/profile.h>
39 #define _KERNEL
40 #include <sys/user.h>
41 #undef _KERNEL
42 #undef sigqueue
43 #include <libutil.h>
44 #endif
45 #endif
46 #else
47 #include "exec/ram_addr.h"
48 #endif
50 #include "exec/cputlb.h"
51 #include "exec/tb-hash.h"
52 #include "translate-all.h"
53 #include "qemu/bitmap.h"
54 #include "qemu/error-report.h"
55 #include "qemu/qemu-print.h"
56 #include "qemu/timer.h"
57 #include "qemu/main-loop.h"
58 #include "exec/log.h"
59 #include "sysemu/cpus.h"
60 #include "sysemu/tcg.h"
62 /* #define DEBUG_TB_INVALIDATE */
63 /* #define DEBUG_TB_FLUSH */
64 /* make various TB consistency checks */
65 /* #define DEBUG_TB_CHECK */
67 #ifdef DEBUG_TB_INVALIDATE
68 #define DEBUG_TB_INVALIDATE_GATE 1
69 #else
70 #define DEBUG_TB_INVALIDATE_GATE 0
71 #endif
73 #ifdef DEBUG_TB_FLUSH
74 #define DEBUG_TB_FLUSH_GATE 1
75 #else
76 #define DEBUG_TB_FLUSH_GATE 0
77 #endif
79 #if !defined(CONFIG_USER_ONLY)
80 /* TB consistency checks only implemented for usermode emulation. */
81 #undef DEBUG_TB_CHECK
82 #endif
84 #ifdef DEBUG_TB_CHECK
85 #define DEBUG_TB_CHECK_GATE 1
86 #else
87 #define DEBUG_TB_CHECK_GATE 0
88 #endif
90 /* Access to the various translations structures need to be serialised via locks
91 * for consistency.
92 * In user-mode emulation access to the memory related structures are protected
93 * with mmap_lock.
94 * In !user-mode we use per-page locks.
96 #ifdef CONFIG_SOFTMMU
97 #define assert_memory_lock()
98 #else
99 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
100 #endif
102 #define SMC_BITMAP_USE_THRESHOLD 10
104 typedef struct PageDesc {
105 /* list of TBs intersecting this ram page */
106 uintptr_t first_tb;
107 #ifdef CONFIG_SOFTMMU
108 /* in order to optimize self modifying code, we count the number
109 of lookups we do to a given page to use a bitmap */
110 unsigned long *code_bitmap;
111 unsigned int code_write_count;
112 #else
113 unsigned long flags;
114 #endif
115 #ifndef CONFIG_USER_ONLY
116 QemuSpin lock;
117 #endif
118 } PageDesc;
121 * struct page_entry - page descriptor entry
122 * @pd: pointer to the &struct PageDesc of the page this entry represents
123 * @index: page index of the page
124 * @locked: whether the page is locked
126 * This struct helps us keep track of the locked state of a page, without
127 * bloating &struct PageDesc.
129 * A page lock protects accesses to all fields of &struct PageDesc.
131 * See also: &struct page_collection.
133 struct page_entry {
134 PageDesc *pd;
135 tb_page_addr_t index;
136 bool locked;
140 * struct page_collection - tracks a set of pages (i.e. &struct page_entry's)
141 * @tree: Binary search tree (BST) of the pages, with key == page index
142 * @max: Pointer to the page in @tree with the highest page index
144 * To avoid deadlock we lock pages in ascending order of page index.
145 * When operating on a set of pages, we need to keep track of them so that
146 * we can lock them in order and also unlock them later. For this we collect
147 * pages (i.e. &struct page_entry's) in a binary search @tree. Given that the
148 * @tree implementation we use does not provide an O(1) operation to obtain the
149 * highest-ranked element, we use @max to keep track of the inserted page
150 * with the highest index. This is valuable because if a page is not in
151 * the tree and its index is higher than @max's, then we can lock it
152 * without breaking the locking order rule.
154 * Note on naming: 'struct page_set' would be shorter, but we already have a few
155 * page_set_*() helpers, so page_collection is used instead to avoid confusion.
157 * See also: page_collection_lock().
159 struct page_collection {
160 GTree *tree;
161 struct page_entry *max;
164 /* list iterators for lists of tagged pointers in TranslationBlock */
165 #define TB_FOR_EACH_TAGGED(head, tb, n, field) \
166 for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
167 tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
168 tb = (TranslationBlock *)((uintptr_t)tb & ~1))
170 #define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
171 TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
173 #define TB_FOR_EACH_JMP(head_tb, tb, n) \
174 TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
177 * In system mode we want L1_MAP to be based on ram offsets,
178 * while in user mode we want it to be based on virtual addresses.
180 * TODO: For user mode, see the caveat re host vs guest virtual
181 * address spaces near GUEST_ADDR_MAX.
183 #if !defined(CONFIG_USER_ONLY)
184 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
185 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
186 #else
187 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
188 #endif
189 #else
190 # define L1_MAP_ADDR_SPACE_BITS MIN(HOST_LONG_BITS, TARGET_ABI_BITS)
191 #endif
193 /* Size of the L2 (and L3, etc) page tables. */
194 #define V_L2_BITS 10
195 #define V_L2_SIZE (1 << V_L2_BITS)
197 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
198 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
199 sizeof_field(TranslationBlock, trace_vcpu_dstate)
200 * BITS_PER_BYTE);
203 * L1 Mapping properties
205 static int v_l1_size;
206 static int v_l1_shift;
207 static int v_l2_levels;
209 /* The bottom level has pointers to PageDesc, and is indexed by
210 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
212 #define V_L1_MIN_BITS 4
213 #define V_L1_MAX_BITS (V_L2_BITS + 3)
214 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
216 static void *l1_map[V_L1_MAX_SIZE];
218 /* code generation context */
219 TCGContext tcg_init_ctx;
220 __thread TCGContext *tcg_ctx;
221 TBContext tb_ctx;
222 bool parallel_cpus;
224 static void page_table_config_init(void)
226 uint32_t v_l1_bits;
228 assert(TARGET_PAGE_BITS);
229 /* The bits remaining after N lower levels of page tables. */
230 v_l1_bits = (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS;
231 if (v_l1_bits < V_L1_MIN_BITS) {
232 v_l1_bits += V_L2_BITS;
235 v_l1_size = 1 << v_l1_bits;
236 v_l1_shift = L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - v_l1_bits;
237 v_l2_levels = v_l1_shift / V_L2_BITS - 1;
239 assert(v_l1_bits <= V_L1_MAX_BITS);
240 assert(v_l1_shift % V_L2_BITS == 0);
241 assert(v_l2_levels >= 0);
244 void cpu_gen_init(void)
246 tcg_context_init(&tcg_init_ctx);
249 /* Encode VAL as a signed leb128 sequence at P.
250 Return P incremented past the encoded value. */
251 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
253 int more, byte;
255 do {
256 byte = val & 0x7f;
257 val >>= 7;
258 more = !((val == 0 && (byte & 0x40) == 0)
259 || (val == -1 && (byte & 0x40) != 0));
260 if (more) {
261 byte |= 0x80;
263 *p++ = byte;
264 } while (more);
266 return p;
269 /* Decode a signed leb128 sequence at *PP; increment *PP past the
270 decoded value. Return the decoded value. */
271 static target_long decode_sleb128(uint8_t **pp)
273 uint8_t *p = *pp;
274 target_long val = 0;
275 int byte, shift = 0;
277 do {
278 byte = *p++;
279 val |= (target_ulong)(byte & 0x7f) << shift;
280 shift += 7;
281 } while (byte & 0x80);
282 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
283 val |= -(target_ulong)1 << shift;
286 *pp = p;
287 return val;
290 /* Encode the data collected about the instructions while compiling TB.
291 Place the data at BLOCK, and return the number of bytes consumed.
293 The logical table consists of TARGET_INSN_START_WORDS target_ulong's,
294 which come from the target's insn_start data, followed by a uintptr_t
295 which comes from the host pc of the end of the code implementing the insn.
297 Each line of the table is encoded as sleb128 deltas from the previous
298 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
299 That is, the first column is seeded with the guest pc, the last column
300 with the host pc, and the middle columns with zeros. */
302 static int encode_search(TranslationBlock *tb, uint8_t *block)
304 uint8_t *highwater = tcg_ctx->code_gen_highwater;
305 uint8_t *p = block;
306 int i, j, n;
308 for (i = 0, n = tb->icount; i < n; ++i) {
309 target_ulong prev;
311 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
312 if (i == 0) {
313 prev = (j == 0 ? tb->pc : 0);
314 } else {
315 prev = tcg_ctx->gen_insn_data[i - 1][j];
317 p = encode_sleb128(p, tcg_ctx->gen_insn_data[i][j] - prev);
319 prev = (i == 0 ? 0 : tcg_ctx->gen_insn_end_off[i - 1]);
320 p = encode_sleb128(p, tcg_ctx->gen_insn_end_off[i] - prev);
322 /* Test for (pending) buffer overflow. The assumption is that any
323 one row beginning below the high water mark cannot overrun
324 the buffer completely. Thus we can test for overflow after
325 encoding a row without having to check during encoding. */
326 if (unlikely(p > highwater)) {
327 return -1;
331 return p - block;
334 /* The cpu state corresponding to 'searched_pc' is restored.
335 * When reset_icount is true, current TB will be interrupted and
336 * icount should be recalculated.
338 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
339 uintptr_t searched_pc, bool reset_icount)
341 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
342 uintptr_t host_pc = (uintptr_t)tb->tc.ptr;
343 CPUArchState *env = cpu->env_ptr;
344 uint8_t *p = tb->tc.ptr + tb->tc.size;
345 int i, j, num_insns = tb->icount;
346 #ifdef CONFIG_PROFILER
347 TCGProfile *prof = &tcg_ctx->prof;
348 int64_t ti = profile_getclock();
349 #endif
351 searched_pc -= GETPC_ADJ;
353 if (searched_pc < host_pc) {
354 return -1;
357 /* Reconstruct the stored insn data while looking for the point at
358 which the end of the insn exceeds the searched_pc. */
359 for (i = 0; i < num_insns; ++i) {
360 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
361 data[j] += decode_sleb128(&p);
363 host_pc += decode_sleb128(&p);
364 if (host_pc > searched_pc) {
365 goto found;
368 return -1;
370 found:
371 if (reset_icount && (tb_cflags(tb) & CF_USE_ICOUNT)) {
372 assert(use_icount);
373 /* Reset the cycle counter to the start of the block
374 and shift if to the number of actually executed instructions */
375 cpu_neg(cpu)->icount_decr.u16.low += num_insns - i;
377 restore_state_to_opc(env, tb, data);
379 #ifdef CONFIG_PROFILER
380 atomic_set(&prof->restore_time,
381 prof->restore_time + profile_getclock() - ti);
382 atomic_set(&prof->restore_count, prof->restore_count + 1);
383 #endif
384 return 0;
387 void tb_destroy(TranslationBlock *tb)
389 qemu_spin_destroy(&tb->jmp_lock);
392 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit)
394 TranslationBlock *tb;
395 bool r = false;
396 uintptr_t check_offset;
398 /* The host_pc has to be in the region of current code buffer. If
399 * it is not we will not be able to resolve it here. The two cases
400 * where host_pc will not be correct are:
402 * - fault during translation (instruction fetch)
403 * - fault from helper (not using GETPC() macro)
405 * Either way we need return early as we can't resolve it here.
407 * We are using unsigned arithmetic so if host_pc <
408 * tcg_init_ctx.code_gen_buffer check_offset will wrap to way
409 * above the code_gen_buffer_size
411 check_offset = host_pc - (uintptr_t) tcg_init_ctx.code_gen_buffer;
413 if (check_offset < tcg_init_ctx.code_gen_buffer_size) {
414 tb = tcg_tb_lookup(host_pc);
415 if (tb) {
416 cpu_restore_state_from_tb(cpu, tb, host_pc, will_exit);
417 if (tb_cflags(tb) & CF_NOCACHE) {
418 /* one-shot translation, invalidate it immediately */
419 tb_phys_invalidate(tb, -1);
420 tcg_tb_remove(tb);
421 tb_destroy(tb);
423 r = true;
427 return r;
430 static void page_init(void)
432 page_size_init();
433 page_table_config_init();
435 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
437 #ifdef HAVE_KINFO_GETVMMAP
438 struct kinfo_vmentry *freep;
439 int i, cnt;
441 freep = kinfo_getvmmap(getpid(), &cnt);
442 if (freep) {
443 mmap_lock();
444 for (i = 0; i < cnt; i++) {
445 unsigned long startaddr, endaddr;
447 startaddr = freep[i].kve_start;
448 endaddr = freep[i].kve_end;
449 if (h2g_valid(startaddr)) {
450 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
452 if (h2g_valid(endaddr)) {
453 endaddr = h2g(endaddr);
454 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
455 } else {
456 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
457 endaddr = ~0ul;
458 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
459 #endif
463 free(freep);
464 mmap_unlock();
466 #else
467 FILE *f;
469 last_brk = (unsigned long)sbrk(0);
471 f = fopen("/compat/linux/proc/self/maps", "r");
472 if (f) {
473 mmap_lock();
475 do {
476 unsigned long startaddr, endaddr;
477 int n;
479 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
481 if (n == 2 && h2g_valid(startaddr)) {
482 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
484 if (h2g_valid(endaddr)) {
485 endaddr = h2g(endaddr);
486 } else {
487 endaddr = ~0ul;
489 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
491 } while (!feof(f));
493 fclose(f);
494 mmap_unlock();
496 #endif
498 #endif
501 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
503 PageDesc *pd;
504 void **lp;
505 int i;
507 /* Level 1. Always allocated. */
508 lp = l1_map + ((index >> v_l1_shift) & (v_l1_size - 1));
510 /* Level 2..N-1. */
511 for (i = v_l2_levels; i > 0; i--) {
512 void **p = atomic_rcu_read(lp);
514 if (p == NULL) {
515 void *existing;
517 if (!alloc) {
518 return NULL;
520 p = g_new0(void *, V_L2_SIZE);
521 existing = atomic_cmpxchg(lp, NULL, p);
522 if (unlikely(existing)) {
523 g_free(p);
524 p = existing;
528 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
531 pd = atomic_rcu_read(lp);
532 if (pd == NULL) {
533 void *existing;
535 if (!alloc) {
536 return NULL;
538 pd = g_new0(PageDesc, V_L2_SIZE);
539 #ifndef CONFIG_USER_ONLY
541 int i;
543 for (i = 0; i < V_L2_SIZE; i++) {
544 qemu_spin_init(&pd[i].lock);
547 #endif
548 existing = atomic_cmpxchg(lp, NULL, pd);
549 if (unlikely(existing)) {
550 #ifndef CONFIG_USER_ONLY
552 int i;
554 for (i = 0; i < V_L2_SIZE; i++) {
555 qemu_spin_destroy(&pd[i].lock);
558 #endif
559 g_free(pd);
560 pd = existing;
564 return pd + (index & (V_L2_SIZE - 1));
567 static inline PageDesc *page_find(tb_page_addr_t index)
569 return page_find_alloc(index, 0);
572 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
573 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc);
575 /* In user-mode page locks aren't used; mmap_lock is enough */
576 #ifdef CONFIG_USER_ONLY
578 #define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
580 static inline void page_lock(PageDesc *pd)
583 static inline void page_unlock(PageDesc *pd)
586 static inline void page_lock_tb(const TranslationBlock *tb)
589 static inline void page_unlock_tb(const TranslationBlock *tb)
592 struct page_collection *
593 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
595 return NULL;
598 void page_collection_unlock(struct page_collection *set)
600 #else /* !CONFIG_USER_ONLY */
602 #ifdef CONFIG_DEBUG_TCG
604 static __thread GHashTable *ht_pages_locked_debug;
606 static void ht_pages_locked_debug_init(void)
608 if (ht_pages_locked_debug) {
609 return;
611 ht_pages_locked_debug = g_hash_table_new(NULL, NULL);
614 static bool page_is_locked(const PageDesc *pd)
616 PageDesc *found;
618 ht_pages_locked_debug_init();
619 found = g_hash_table_lookup(ht_pages_locked_debug, pd);
620 return !!found;
623 static void page_lock__debug(PageDesc *pd)
625 ht_pages_locked_debug_init();
626 g_assert(!page_is_locked(pd));
627 g_hash_table_insert(ht_pages_locked_debug, pd, pd);
630 static void page_unlock__debug(const PageDesc *pd)
632 bool removed;
634 ht_pages_locked_debug_init();
635 g_assert(page_is_locked(pd));
636 removed = g_hash_table_remove(ht_pages_locked_debug, pd);
637 g_assert(removed);
640 static void
641 do_assert_page_locked(const PageDesc *pd, const char *file, int line)
643 if (unlikely(!page_is_locked(pd))) {
644 error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
645 pd, file, line);
646 abort();
650 #define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
652 void assert_no_pages_locked(void)
654 ht_pages_locked_debug_init();
655 g_assert(g_hash_table_size(ht_pages_locked_debug) == 0);
658 #else /* !CONFIG_DEBUG_TCG */
660 #define assert_page_locked(pd)
662 static inline void page_lock__debug(const PageDesc *pd)
666 static inline void page_unlock__debug(const PageDesc *pd)
670 #endif /* CONFIG_DEBUG_TCG */
672 static inline void page_lock(PageDesc *pd)
674 page_lock__debug(pd);
675 qemu_spin_lock(&pd->lock);
678 static inline void page_unlock(PageDesc *pd)
680 qemu_spin_unlock(&pd->lock);
681 page_unlock__debug(pd);
684 /* lock the page(s) of a TB in the correct acquisition order */
685 static inline void page_lock_tb(const TranslationBlock *tb)
687 page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], 0);
690 static inline void page_unlock_tb(const TranslationBlock *tb)
692 PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
694 page_unlock(p1);
695 if (unlikely(tb->page_addr[1] != -1)) {
696 PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
698 if (p2 != p1) {
699 page_unlock(p2);
704 static inline struct page_entry *
705 page_entry_new(PageDesc *pd, tb_page_addr_t index)
707 struct page_entry *pe = g_malloc(sizeof(*pe));
709 pe->index = index;
710 pe->pd = pd;
711 pe->locked = false;
712 return pe;
715 static void page_entry_destroy(gpointer p)
717 struct page_entry *pe = p;
719 g_assert(pe->locked);
720 page_unlock(pe->pd);
721 g_free(pe);
724 /* returns false on success */
725 static bool page_entry_trylock(struct page_entry *pe)
727 bool busy;
729 busy = qemu_spin_trylock(&pe->pd->lock);
730 if (!busy) {
731 g_assert(!pe->locked);
732 pe->locked = true;
733 page_lock__debug(pe->pd);
735 return busy;
738 static void do_page_entry_lock(struct page_entry *pe)
740 page_lock(pe->pd);
741 g_assert(!pe->locked);
742 pe->locked = true;
745 static gboolean page_entry_lock(gpointer key, gpointer value, gpointer data)
747 struct page_entry *pe = value;
749 do_page_entry_lock(pe);
750 return FALSE;
753 static gboolean page_entry_unlock(gpointer key, gpointer value, gpointer data)
755 struct page_entry *pe = value;
757 if (pe->locked) {
758 pe->locked = false;
759 page_unlock(pe->pd);
761 return FALSE;
765 * Trylock a page, and if successful, add the page to a collection.
766 * Returns true ("busy") if the page could not be locked; false otherwise.
768 static bool page_trylock_add(struct page_collection *set, tb_page_addr_t addr)
770 tb_page_addr_t index = addr >> TARGET_PAGE_BITS;
771 struct page_entry *pe;
772 PageDesc *pd;
774 pe = g_tree_lookup(set->tree, &index);
775 if (pe) {
776 return false;
779 pd = page_find(index);
780 if (pd == NULL) {
781 return false;
784 pe = page_entry_new(pd, index);
785 g_tree_insert(set->tree, &pe->index, pe);
788 * If this is either (1) the first insertion or (2) a page whose index
789 * is higher than any other so far, just lock the page and move on.
791 if (set->max == NULL || pe->index > set->max->index) {
792 set->max = pe;
793 do_page_entry_lock(pe);
794 return false;
797 * Try to acquire out-of-order lock; if busy, return busy so that we acquire
798 * locks in order.
800 return page_entry_trylock(pe);
803 static gint tb_page_addr_cmp(gconstpointer ap, gconstpointer bp, gpointer udata)
805 tb_page_addr_t a = *(const tb_page_addr_t *)ap;
806 tb_page_addr_t b = *(const tb_page_addr_t *)bp;
808 if (a == b) {
809 return 0;
810 } else if (a < b) {
811 return -1;
813 return 1;
817 * Lock a range of pages ([@start,@end[) as well as the pages of all
818 * intersecting TBs.
819 * Locking order: acquire locks in ascending order of page index.
821 struct page_collection *
822 page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
824 struct page_collection *set = g_malloc(sizeof(*set));
825 tb_page_addr_t index;
826 PageDesc *pd;
828 start >>= TARGET_PAGE_BITS;
829 end >>= TARGET_PAGE_BITS;
830 g_assert(start <= end);
832 set->tree = g_tree_new_full(tb_page_addr_cmp, NULL, NULL,
833 page_entry_destroy);
834 set->max = NULL;
835 assert_no_pages_locked();
837 retry:
838 g_tree_foreach(set->tree, page_entry_lock, NULL);
840 for (index = start; index <= end; index++) {
841 TranslationBlock *tb;
842 int n;
844 pd = page_find(index);
845 if (pd == NULL) {
846 continue;
848 if (page_trylock_add(set, index << TARGET_PAGE_BITS)) {
849 g_tree_foreach(set->tree, page_entry_unlock, NULL);
850 goto retry;
852 assert_page_locked(pd);
853 PAGE_FOR_EACH_TB(pd, tb, n) {
854 if (page_trylock_add(set, tb->page_addr[0]) ||
855 (tb->page_addr[1] != -1 &&
856 page_trylock_add(set, tb->page_addr[1]))) {
857 /* drop all locks, and reacquire in order */
858 g_tree_foreach(set->tree, page_entry_unlock, NULL);
859 goto retry;
863 return set;
866 void page_collection_unlock(struct page_collection *set)
868 /* entries are unlocked and freed via page_entry_destroy */
869 g_tree_destroy(set->tree);
870 g_free(set);
873 #endif /* !CONFIG_USER_ONLY */
875 static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
876 PageDesc **ret_p2, tb_page_addr_t phys2, int alloc)
878 PageDesc *p1, *p2;
879 tb_page_addr_t page1;
880 tb_page_addr_t page2;
882 assert_memory_lock();
883 g_assert(phys1 != -1);
885 page1 = phys1 >> TARGET_PAGE_BITS;
886 page2 = phys2 >> TARGET_PAGE_BITS;
888 p1 = page_find_alloc(page1, alloc);
889 if (ret_p1) {
890 *ret_p1 = p1;
892 if (likely(phys2 == -1)) {
893 page_lock(p1);
894 return;
895 } else if (page1 == page2) {
896 page_lock(p1);
897 if (ret_p2) {
898 *ret_p2 = p1;
900 return;
902 p2 = page_find_alloc(page2, alloc);
903 if (ret_p2) {
904 *ret_p2 = p2;
906 if (page1 < page2) {
907 page_lock(p1);
908 page_lock(p2);
909 } else {
910 page_lock(p2);
911 page_lock(p1);
915 /* Minimum size of the code gen buffer. This number is randomly chosen,
916 but not so small that we can't have a fair number of TB's live. */
917 #define MIN_CODE_GEN_BUFFER_SIZE (1 * MiB)
919 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
920 indicated, this is constrained by the range of direct branches on the
921 host cpu, as used by the TCG implementation of goto_tb. */
922 #if defined(__x86_64__)
923 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
924 #elif defined(__sparc__)
925 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
926 #elif defined(__powerpc64__)
927 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
928 #elif defined(__powerpc__)
929 # define MAX_CODE_GEN_BUFFER_SIZE (32 * MiB)
930 #elif defined(__aarch64__)
931 # define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB)
932 #elif defined(__s390x__)
933 /* We have a +- 4GB range on the branches; leave some slop. */
934 # define MAX_CODE_GEN_BUFFER_SIZE (3 * GiB)
935 #elif defined(__mips__)
936 /* We have a 256MB branch region, but leave room to make sure the
937 main executable is also within that region. */
938 # define MAX_CODE_GEN_BUFFER_SIZE (128 * MiB)
939 #else
940 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
941 #endif
943 #if TCG_TARGET_REG_BITS == 32
944 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32 * MiB)
945 #ifdef CONFIG_USER_ONLY
947 * For user mode on smaller 32 bit systems we may run into trouble
948 * allocating big chunks of data in the right place. On these systems
949 * we utilise a static code generation buffer directly in the binary.
951 #define USE_STATIC_CODE_GEN_BUFFER
952 #endif
953 #else /* TCG_TARGET_REG_BITS == 64 */
954 #ifdef CONFIG_USER_ONLY
956 * As user-mode emulation typically means running multiple instances
957 * of the translator don't go too nuts with our default code gen
958 * buffer lest we make things too hard for the OS.
960 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (128 * MiB)
961 #else
963 * We expect most system emulation to run one or two guests per host.
964 * Users running large scale system emulation may want to tweak their
965 * runtime setup via the tb-size control on the command line.
967 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (1 * GiB)
968 #endif
969 #endif
971 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
972 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
973 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
975 static inline size_t size_code_gen_buffer(size_t tb_size)
977 /* Size the buffer. */
978 if (tb_size == 0) {
979 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
981 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
982 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
984 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
985 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
987 return tb_size;
990 #ifdef __mips__
991 /* In order to use J and JAL within the code_gen_buffer, we require
992 that the buffer not cross a 256MB boundary. */
993 static inline bool cross_256mb(void *addr, size_t size)
995 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
998 /* We weren't able to allocate a buffer without crossing that boundary,
999 so make do with the larger portion of the buffer that doesn't cross.
1000 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
1001 static inline void *split_cross_256mb(void *buf1, size_t size1)
1003 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
1004 size_t size2 = buf1 + size1 - buf2;
1006 size1 = buf2 - buf1;
1007 if (size1 < size2) {
1008 size1 = size2;
1009 buf1 = buf2;
1012 tcg_ctx->code_gen_buffer_size = size1;
1013 return buf1;
1015 #endif
1017 #ifdef USE_STATIC_CODE_GEN_BUFFER
1018 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
1019 __attribute__((aligned(CODE_GEN_ALIGN)));
1021 static inline void *alloc_code_gen_buffer(void)
1023 void *buf = static_code_gen_buffer;
1024 void *end = static_code_gen_buffer + sizeof(static_code_gen_buffer);
1025 size_t size;
1027 /* page-align the beginning and end of the buffer */
1028 buf = QEMU_ALIGN_PTR_UP(buf, qemu_real_host_page_size);
1029 end = QEMU_ALIGN_PTR_DOWN(end, qemu_real_host_page_size);
1031 size = end - buf;
1033 /* Honor a command-line option limiting the size of the buffer. */
1034 if (size > tcg_ctx->code_gen_buffer_size) {
1035 size = QEMU_ALIGN_DOWN(tcg_ctx->code_gen_buffer_size,
1036 qemu_real_host_page_size);
1038 tcg_ctx->code_gen_buffer_size = size;
1040 #ifdef __mips__
1041 if (cross_256mb(buf, size)) {
1042 buf = split_cross_256mb(buf, size);
1043 size = tcg_ctx->code_gen_buffer_size;
1045 #endif
1047 if (qemu_mprotect_rwx(buf, size)) {
1048 abort();
1050 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1052 return buf;
1054 #elif defined(_WIN32)
1055 static inline void *alloc_code_gen_buffer(void)
1057 size_t size = tcg_ctx->code_gen_buffer_size;
1058 return VirtualAlloc(NULL, size, MEM_RESERVE | MEM_COMMIT,
1059 PAGE_EXECUTE_READWRITE);
1061 #else
1062 static inline void *alloc_code_gen_buffer(void)
1064 int prot = PROT_WRITE | PROT_READ | PROT_EXEC;
1065 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
1066 size_t size = tcg_ctx->code_gen_buffer_size;
1067 void *buf;
1069 buf = mmap(NULL, size, prot, flags, -1, 0);
1070 if (buf == MAP_FAILED) {
1071 return NULL;
1074 #ifdef __mips__
1075 if (cross_256mb(buf, size)) {
1077 * Try again, with the original still mapped, to avoid re-acquiring
1078 * the same 256mb crossing.
1080 size_t size2;
1081 void *buf2 = mmap(NULL, size, prot, flags, -1, 0);
1082 switch ((int)(buf2 != MAP_FAILED)) {
1083 case 1:
1084 if (!cross_256mb(buf2, size)) {
1085 /* Success! Use the new buffer. */
1086 munmap(buf, size);
1087 break;
1089 /* Failure. Work with what we had. */
1090 munmap(buf2, size);
1091 /* fallthru */
1092 default:
1093 /* Split the original buffer. Free the smaller half. */
1094 buf2 = split_cross_256mb(buf, size);
1095 size2 = tcg_ctx->code_gen_buffer_size;
1096 if (buf == buf2) {
1097 munmap(buf + size2, size - size2);
1098 } else {
1099 munmap(buf, size - size2);
1101 size = size2;
1102 break;
1104 buf = buf2;
1106 #endif
1108 /* Request large pages for the buffer. */
1109 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
1111 return buf;
1113 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
1115 static inline void code_gen_alloc(size_t tb_size)
1117 tcg_ctx->code_gen_buffer_size = size_code_gen_buffer(tb_size);
1118 tcg_ctx->code_gen_buffer = alloc_code_gen_buffer();
1119 if (tcg_ctx->code_gen_buffer == NULL) {
1120 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
1121 exit(1);
1125 static bool tb_cmp(const void *ap, const void *bp)
1127 const TranslationBlock *a = ap;
1128 const TranslationBlock *b = bp;
1130 return a->pc == b->pc &&
1131 a->cs_base == b->cs_base &&
1132 a->flags == b->flags &&
1133 (tb_cflags(a) & CF_HASH_MASK) == (tb_cflags(b) & CF_HASH_MASK) &&
1134 a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
1135 a->page_addr[0] == b->page_addr[0] &&
1136 a->page_addr[1] == b->page_addr[1];
1139 static void tb_htable_init(void)
1141 unsigned int mode = QHT_MODE_AUTO_RESIZE;
1143 qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
1146 /* Must be called before using the QEMU cpus. 'tb_size' is the size
1147 (in bytes) allocated to the translation buffer. Zero means default
1148 size. */
1149 void tcg_exec_init(unsigned long tb_size)
1151 tcg_allowed = true;
1152 cpu_gen_init();
1153 page_init();
1154 tb_htable_init();
1155 code_gen_alloc(tb_size);
1156 #if defined(CONFIG_SOFTMMU)
1157 /* There's no guest base to take into account, so go ahead and
1158 initialize the prologue now. */
1159 tcg_prologue_init(tcg_ctx);
1160 #endif
1163 /* call with @p->lock held */
1164 static inline void invalidate_page_bitmap(PageDesc *p)
1166 assert_page_locked(p);
1167 #ifdef CONFIG_SOFTMMU
1168 g_free(p->code_bitmap);
1169 p->code_bitmap = NULL;
1170 p->code_write_count = 0;
1171 #endif
1174 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
1175 static void page_flush_tb_1(int level, void **lp)
1177 int i;
1179 if (*lp == NULL) {
1180 return;
1182 if (level == 0) {
1183 PageDesc *pd = *lp;
1185 for (i = 0; i < V_L2_SIZE; ++i) {
1186 page_lock(&pd[i]);
1187 pd[i].first_tb = (uintptr_t)NULL;
1188 invalidate_page_bitmap(pd + i);
1189 page_unlock(&pd[i]);
1191 } else {
1192 void **pp = *lp;
1194 for (i = 0; i < V_L2_SIZE; ++i) {
1195 page_flush_tb_1(level - 1, pp + i);
1200 static void page_flush_tb(void)
1202 int i, l1_sz = v_l1_size;
1204 for (i = 0; i < l1_sz; i++) {
1205 page_flush_tb_1(v_l2_levels, l1_map + i);
1209 static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data)
1211 const TranslationBlock *tb = value;
1212 size_t *size = data;
1214 *size += tb->tc.size;
1215 return false;
1218 /* flush all the translation blocks */
1219 static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
1221 bool did_flush = false;
1223 mmap_lock();
1224 /* If it is already been done on request of another CPU,
1225 * just retry.
1227 if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
1228 goto done;
1230 did_flush = true;
1232 if (DEBUG_TB_FLUSH_GATE) {
1233 size_t nb_tbs = tcg_nb_tbs();
1234 size_t host_size = 0;
1236 tcg_tb_foreach(tb_host_size_iter, &host_size);
1237 printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
1238 tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0);
1241 CPU_FOREACH(cpu) {
1242 cpu_tb_jmp_cache_clear(cpu);
1245 qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
1246 page_flush_tb();
1248 tcg_region_reset_all();
1249 /* XXX: flush processor icache at this point if cache flush is
1250 expensive */
1251 atomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
1253 done:
1254 mmap_unlock();
1255 if (did_flush) {
1256 qemu_plugin_flush_cb();
1260 void tb_flush(CPUState *cpu)
1262 if (tcg_enabled()) {
1263 unsigned tb_flush_count = atomic_mb_read(&tb_ctx.tb_flush_count);
1265 if (cpu_in_exclusive_context(cpu)) {
1266 do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
1267 } else {
1268 async_safe_run_on_cpu(cpu, do_tb_flush,
1269 RUN_ON_CPU_HOST_INT(tb_flush_count));
1275 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
1276 * so in order to prevent bit rot we compile them unconditionally in user-mode,
1277 * and let the optimizer get rid of them by wrapping their user-only callers
1278 * with if (DEBUG_TB_CHECK_GATE).
1280 #ifdef CONFIG_USER_ONLY
1282 static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp)
1284 TranslationBlock *tb = p;
1285 target_ulong addr = *(target_ulong *)userp;
1287 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
1288 printf("ERROR invalidate: address=" TARGET_FMT_lx
1289 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
1293 /* verify that all the pages have correct rights for code
1295 * Called with mmap_lock held.
1297 static void tb_invalidate_check(target_ulong address)
1299 address &= TARGET_PAGE_MASK;
1300 qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address);
1303 static void do_tb_page_check(void *p, uint32_t hash, void *userp)
1305 TranslationBlock *tb = p;
1306 int flags1, flags2;
1308 flags1 = page_get_flags(tb->pc);
1309 flags2 = page_get_flags(tb->pc + tb->size - 1);
1310 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
1311 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
1312 (long)tb->pc, tb->size, flags1, flags2);
1316 /* verify that all the pages have correct rights for code */
1317 static void tb_page_check(void)
1319 qht_iter(&tb_ctx.htable, do_tb_page_check, NULL);
1322 #endif /* CONFIG_USER_ONLY */
1325 * user-mode: call with mmap_lock held
1326 * !user-mode: call with @pd->lock held
1328 static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
1330 TranslationBlock *tb1;
1331 uintptr_t *pprev;
1332 unsigned int n1;
1334 assert_page_locked(pd);
1335 pprev = &pd->first_tb;
1336 PAGE_FOR_EACH_TB(pd, tb1, n1) {
1337 if (tb1 == tb) {
1338 *pprev = tb1->page_next[n1];
1339 return;
1341 pprev = &tb1->page_next[n1];
1343 g_assert_not_reached();
1346 /* remove @orig from its @n_orig-th jump list */
1347 static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
1349 uintptr_t ptr, ptr_locked;
1350 TranslationBlock *dest;
1351 TranslationBlock *tb;
1352 uintptr_t *pprev;
1353 int n;
1355 /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
1356 ptr = atomic_or_fetch(&orig->jmp_dest[n_orig], 1);
1357 dest = (TranslationBlock *)(ptr & ~1);
1358 if (dest == NULL) {
1359 return;
1362 qemu_spin_lock(&dest->jmp_lock);
1364 * While acquiring the lock, the jump might have been removed if the
1365 * destination TB was invalidated; check again.
1367 ptr_locked = atomic_read(&orig->jmp_dest[n_orig]);
1368 if (ptr_locked != ptr) {
1369 qemu_spin_unlock(&dest->jmp_lock);
1371 * The only possibility is that the jump was unlinked via
1372 * tb_jump_unlink(dest). Seeing here another destination would be a bug,
1373 * because we set the LSB above.
1375 g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
1376 return;
1379 * We first acquired the lock, and since the destination pointer matches,
1380 * we know for sure that @orig is in the jmp list.
1382 pprev = &dest->jmp_list_head;
1383 TB_FOR_EACH_JMP(dest, tb, n) {
1384 if (tb == orig && n == n_orig) {
1385 *pprev = tb->jmp_list_next[n];
1386 /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
1387 qemu_spin_unlock(&dest->jmp_lock);
1388 return;
1390 pprev = &tb->jmp_list_next[n];
1392 g_assert_not_reached();
1395 /* reset the jump entry 'n' of a TB so that it is not chained to
1396 another TB */
1397 static inline void tb_reset_jump(TranslationBlock *tb, int n)
1399 uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
1400 tb_set_jmp_target(tb, n, addr);
1403 /* remove any jumps to the TB */
1404 static inline void tb_jmp_unlink(TranslationBlock *dest)
1406 TranslationBlock *tb;
1407 int n;
1409 qemu_spin_lock(&dest->jmp_lock);
1411 TB_FOR_EACH_JMP(dest, tb, n) {
1412 tb_reset_jump(tb, n);
1413 atomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
1414 /* No need to clear the list entry; setting the dest ptr is enough */
1416 dest->jmp_list_head = (uintptr_t)NULL;
1418 qemu_spin_unlock(&dest->jmp_lock);
1422 * In user-mode, call with mmap_lock held.
1423 * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
1424 * locks held.
1426 static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
1428 CPUState *cpu;
1429 PageDesc *p;
1430 uint32_t h;
1431 tb_page_addr_t phys_pc;
1433 assert_memory_lock();
1435 /* make sure no further incoming jumps will be chained to this TB */
1436 qemu_spin_lock(&tb->jmp_lock);
1437 atomic_set(&tb->cflags, tb->cflags | CF_INVALID);
1438 qemu_spin_unlock(&tb->jmp_lock);
1440 /* remove the TB from the hash list */
1441 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1442 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb_cflags(tb) & CF_HASH_MASK,
1443 tb->trace_vcpu_dstate);
1444 if (!(tb->cflags & CF_NOCACHE) &&
1445 !qht_remove(&tb_ctx.htable, tb, h)) {
1446 return;
1449 /* remove the TB from the page list */
1450 if (rm_from_page_list) {
1451 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1452 tb_page_remove(p, tb);
1453 invalidate_page_bitmap(p);
1454 if (tb->page_addr[1] != -1) {
1455 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1456 tb_page_remove(p, tb);
1457 invalidate_page_bitmap(p);
1461 /* remove the TB from the hash list */
1462 h = tb_jmp_cache_hash_func(tb->pc);
1463 CPU_FOREACH(cpu) {
1464 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) {
1465 atomic_set(&cpu->tb_jmp_cache[h], NULL);
1469 /* suppress this TB from the two jump lists */
1470 tb_remove_from_jmp_list(tb, 0);
1471 tb_remove_from_jmp_list(tb, 1);
1473 /* suppress any remaining jumps to this TB */
1474 tb_jmp_unlink(tb);
1476 atomic_set(&tcg_ctx->tb_phys_invalidate_count,
1477 tcg_ctx->tb_phys_invalidate_count + 1);
1480 static void tb_phys_invalidate__locked(TranslationBlock *tb)
1482 do_tb_phys_invalidate(tb, true);
1485 /* invalidate one TB
1487 * Called with mmap_lock held in user-mode.
1489 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1491 if (page_addr == -1 && tb->page_addr[0] != -1) {
1492 page_lock_tb(tb);
1493 do_tb_phys_invalidate(tb, true);
1494 page_unlock_tb(tb);
1495 } else {
1496 do_tb_phys_invalidate(tb, false);
1500 #ifdef CONFIG_SOFTMMU
1501 /* call with @p->lock held */
1502 static void build_page_bitmap(PageDesc *p)
1504 int n, tb_start, tb_end;
1505 TranslationBlock *tb;
1507 assert_page_locked(p);
1508 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1510 PAGE_FOR_EACH_TB(p, tb, n) {
1511 /* NOTE: this is subtle as a TB may span two physical pages */
1512 if (n == 0) {
1513 /* NOTE: tb_end may be after the end of the page, but
1514 it is not a problem */
1515 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1516 tb_end = tb_start + tb->size;
1517 if (tb_end > TARGET_PAGE_SIZE) {
1518 tb_end = TARGET_PAGE_SIZE;
1520 } else {
1521 tb_start = 0;
1522 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1524 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1527 #endif
1529 /* add the tb in the target page and protect it if necessary
1531 * Called with mmap_lock held for user-mode emulation.
1532 * Called with @p->lock held in !user-mode.
1534 static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
1535 unsigned int n, tb_page_addr_t page_addr)
1537 #ifndef CONFIG_USER_ONLY
1538 bool page_already_protected;
1539 #endif
1541 assert_page_locked(p);
1543 tb->page_addr[n] = page_addr;
1544 tb->page_next[n] = p->first_tb;
1545 #ifndef CONFIG_USER_ONLY
1546 page_already_protected = p->first_tb != (uintptr_t)NULL;
1547 #endif
1548 p->first_tb = (uintptr_t)tb | n;
1549 invalidate_page_bitmap(p);
1551 #if defined(CONFIG_USER_ONLY)
1552 if (p->flags & PAGE_WRITE) {
1553 target_ulong addr;
1554 PageDesc *p2;
1555 int prot;
1557 /* force the host page as non writable (writes will have a
1558 page fault + mprotect overhead) */
1559 page_addr &= qemu_host_page_mask;
1560 prot = 0;
1561 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1562 addr += TARGET_PAGE_SIZE) {
1564 p2 = page_find(addr >> TARGET_PAGE_BITS);
1565 if (!p2) {
1566 continue;
1568 prot |= p2->flags;
1569 p2->flags &= ~PAGE_WRITE;
1571 mprotect(g2h(page_addr), qemu_host_page_size,
1572 (prot & PAGE_BITS) & ~PAGE_WRITE);
1573 if (DEBUG_TB_INVALIDATE_GATE) {
1574 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr);
1577 #else
1578 /* if some code is already present, then the pages are already
1579 protected. So we handle the case where only the first TB is
1580 allocated in a physical page */
1581 if (!page_already_protected) {
1582 tlb_protect_code(page_addr);
1584 #endif
1587 /* add a new TB and link it to the physical page tables. phys_page2 is
1588 * (-1) to indicate that only one page contains the TB.
1590 * Called with mmap_lock held for user-mode emulation.
1592 * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
1593 * Note that in !user-mode, another thread might have already added a TB
1594 * for the same block of guest code that @tb corresponds to. In that case,
1595 * the caller should discard the original @tb, and use instead the returned TB.
1597 static TranslationBlock *
1598 tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1599 tb_page_addr_t phys_page2)
1601 PageDesc *p;
1602 PageDesc *p2 = NULL;
1604 assert_memory_lock();
1606 if (phys_pc == -1) {
1608 * If the TB is not associated with a physical RAM page then
1609 * it must be a temporary one-insn TB, and we have nothing to do
1610 * except fill in the page_addr[] fields.
1612 assert(tb->cflags & CF_NOCACHE);
1613 tb->page_addr[0] = tb->page_addr[1] = -1;
1614 return tb;
1618 * Add the TB to the page list, acquiring first the pages's locks.
1619 * We keep the locks held until after inserting the TB in the hash table,
1620 * so that if the insertion fails we know for sure that the TBs are still
1621 * in the page descriptors.
1622 * Note that inserting into the hash table first isn't an option, since
1623 * we can only insert TBs that are fully initialized.
1625 page_lock_pair(&p, phys_pc, &p2, phys_page2, 1);
1626 tb_page_add(p, tb, 0, phys_pc & TARGET_PAGE_MASK);
1627 if (p2) {
1628 tb_page_add(p2, tb, 1, phys_page2);
1629 } else {
1630 tb->page_addr[1] = -1;
1633 if (!(tb->cflags & CF_NOCACHE)) {
1634 void *existing_tb = NULL;
1635 uint32_t h;
1637 /* add in the hash table */
1638 h = tb_hash_func(phys_pc, tb->pc, tb->flags, tb->cflags & CF_HASH_MASK,
1639 tb->trace_vcpu_dstate);
1640 qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
1642 /* remove TB from the page(s) if we couldn't insert it */
1643 if (unlikely(existing_tb)) {
1644 tb_page_remove(p, tb);
1645 invalidate_page_bitmap(p);
1646 if (p2) {
1647 tb_page_remove(p2, tb);
1648 invalidate_page_bitmap(p2);
1650 tb = existing_tb;
1654 if (p2 && p2 != p) {
1655 page_unlock(p2);
1657 page_unlock(p);
1659 #ifdef CONFIG_USER_ONLY
1660 if (DEBUG_TB_CHECK_GATE) {
1661 tb_page_check();
1663 #endif
1664 return tb;
1667 /* Called with mmap_lock held for user mode emulation. */
1668 TranslationBlock *tb_gen_code(CPUState *cpu,
1669 target_ulong pc, target_ulong cs_base,
1670 uint32_t flags, int cflags)
1672 CPUArchState *env = cpu->env_ptr;
1673 TranslationBlock *tb, *existing_tb;
1674 tb_page_addr_t phys_pc, phys_page2;
1675 target_ulong virt_page2;
1676 tcg_insn_unit *gen_code_buf;
1677 int gen_code_size, search_size, max_insns;
1678 #ifdef CONFIG_PROFILER
1679 TCGProfile *prof = &tcg_ctx->prof;
1680 int64_t ti;
1681 #endif
1683 assert_memory_lock();
1685 phys_pc = get_page_addr_code(env, pc);
1687 if (phys_pc == -1) {
1688 /* Generate a temporary TB with 1 insn in it */
1689 cflags &= ~CF_COUNT_MASK;
1690 cflags |= CF_NOCACHE | 1;
1693 cflags &= ~CF_CLUSTER_MASK;
1694 cflags |= cpu->cluster_index << CF_CLUSTER_SHIFT;
1696 max_insns = cflags & CF_COUNT_MASK;
1697 if (max_insns == 0) {
1698 max_insns = CF_COUNT_MASK;
1700 if (max_insns > TCG_MAX_INSNS) {
1701 max_insns = TCG_MAX_INSNS;
1703 if (cpu->singlestep_enabled || singlestep) {
1704 max_insns = 1;
1707 buffer_overflow:
1708 tb = tcg_tb_alloc(tcg_ctx);
1709 if (unlikely(!tb)) {
1710 /* flush must be done */
1711 tb_flush(cpu);
1712 mmap_unlock();
1713 /* Make the execution loop process the flush as soon as possible. */
1714 cpu->exception_index = EXCP_INTERRUPT;
1715 cpu_loop_exit(cpu);
1718 gen_code_buf = tcg_ctx->code_gen_ptr;
1719 tb->tc.ptr = gen_code_buf;
1720 tb->pc = pc;
1721 tb->cs_base = cs_base;
1722 tb->flags = flags;
1723 tb->cflags = cflags;
1724 tb->orig_tb = NULL;
1725 tb->trace_vcpu_dstate = *cpu->trace_dstate;
1726 tcg_ctx->tb_cflags = cflags;
1727 tb_overflow:
1729 #ifdef CONFIG_PROFILER
1730 /* includes aborted translations because of exceptions */
1731 atomic_set(&prof->tb_count1, prof->tb_count1 + 1);
1732 ti = profile_getclock();
1733 #endif
1735 tcg_func_start(tcg_ctx);
1737 tcg_ctx->cpu = env_cpu(env);
1738 gen_intermediate_code(cpu, tb, max_insns);
1739 tcg_ctx->cpu = NULL;
1741 trace_translate_block(tb, tb->pc, tb->tc.ptr);
1743 /* generate machine code */
1744 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1745 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1746 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
1747 if (TCG_TARGET_HAS_direct_jump) {
1748 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
1749 tcg_ctx->tb_jmp_target_addr = NULL;
1750 } else {
1751 tcg_ctx->tb_jmp_insn_offset = NULL;
1752 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
1755 #ifdef CONFIG_PROFILER
1756 atomic_set(&prof->tb_count, prof->tb_count + 1);
1757 atomic_set(&prof->interm_time, prof->interm_time + profile_getclock() - ti);
1758 ti = profile_getclock();
1759 #endif
1761 gen_code_size = tcg_gen_code(tcg_ctx, tb);
1762 if (unlikely(gen_code_size < 0)) {
1763 switch (gen_code_size) {
1764 case -1:
1766 * Overflow of code_gen_buffer, or the current slice of it.
1768 * TODO: We don't need to re-do gen_intermediate_code, nor
1769 * should we re-do the tcg optimization currently hidden
1770 * inside tcg_gen_code. All that should be required is to
1771 * flush the TBs, allocate a new TB, re-initialize it per
1772 * above, and re-do the actual code generation.
1774 goto buffer_overflow;
1776 case -2:
1778 * The code generated for the TranslationBlock is too large.
1779 * The maximum size allowed by the unwind info is 64k.
1780 * There may be stricter constraints from relocations
1781 * in the tcg backend.
1783 * Try again with half as many insns as we attempted this time.
1784 * If a single insn overflows, there's a bug somewhere...
1786 max_insns = tb->icount;
1787 assert(max_insns > 1);
1788 max_insns /= 2;
1789 goto tb_overflow;
1791 default:
1792 g_assert_not_reached();
1795 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1796 if (unlikely(search_size < 0)) {
1797 goto buffer_overflow;
1799 tb->tc.size = gen_code_size;
1801 #ifdef CONFIG_PROFILER
1802 atomic_set(&prof->code_time, prof->code_time + profile_getclock() - ti);
1803 atomic_set(&prof->code_in_len, prof->code_in_len + tb->size);
1804 atomic_set(&prof->code_out_len, prof->code_out_len + gen_code_size);
1805 atomic_set(&prof->search_out_len, prof->search_out_len + search_size);
1806 #endif
1808 #ifdef DEBUG_DISAS
1809 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1810 qemu_log_in_addr_range(tb->pc)) {
1811 FILE *logfile = qemu_log_lock();
1812 int code_size, data_size = 0;
1813 g_autoptr(GString) note = g_string_new("[tb header & initial instruction]");
1814 size_t chunk_start = 0;
1815 int insn = 0;
1816 qemu_log("OUT: [size=%d]\n", gen_code_size);
1817 if (tcg_ctx->data_gen_ptr) {
1818 code_size = tcg_ctx->data_gen_ptr - tb->tc.ptr;
1819 data_size = gen_code_size - code_size;
1820 } else {
1821 code_size = gen_code_size;
1824 /* Dump header and the first instruction */
1825 chunk_start = tcg_ctx->gen_insn_end_off[insn];
1826 log_disas(tb->tc.ptr, chunk_start, note->str);
1829 * Dump each instruction chunk, wrapping up empty chunks into
1830 * the next instruction. The whole array is offset so the
1831 * first entry is the beginning of the 2nd instruction.
1833 while (insn <= tb->icount && chunk_start < code_size) {
1834 size_t chunk_end = tcg_ctx->gen_insn_end_off[insn];
1835 if (chunk_end > chunk_start) {
1836 g_string_printf(note, "[guest addr: " TARGET_FMT_lx "]",
1837 tcg_ctx->gen_insn_data[insn][0]);
1838 log_disas(tb->tc.ptr + chunk_start, chunk_end - chunk_start,
1839 note->str);
1840 chunk_start = chunk_end;
1842 insn++;
1845 /* Finally dump any data we may have after the block */
1846 if (data_size) {
1847 int i;
1848 qemu_log(" data: [size=%d]\n", data_size);
1849 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1850 if (sizeof(tcg_target_ulong) == 8) {
1851 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1852 (uintptr_t)tcg_ctx->data_gen_ptr + i,
1853 *(uint64_t *)(tcg_ctx->data_gen_ptr + i));
1854 } else {
1855 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
1856 (uintptr_t)tcg_ctx->data_gen_ptr + i,
1857 *(uint32_t *)(tcg_ctx->data_gen_ptr + i));
1861 qemu_log("\n");
1862 qemu_log_flush();
1863 qemu_log_unlock(logfile);
1865 #endif
1867 atomic_set(&tcg_ctx->code_gen_ptr, (void *)
1868 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1869 CODE_GEN_ALIGN));
1871 /* init jump list */
1872 qemu_spin_init(&tb->jmp_lock);
1873 tb->jmp_list_head = (uintptr_t)NULL;
1874 tb->jmp_list_next[0] = (uintptr_t)NULL;
1875 tb->jmp_list_next[1] = (uintptr_t)NULL;
1876 tb->jmp_dest[0] = (uintptr_t)NULL;
1877 tb->jmp_dest[1] = (uintptr_t)NULL;
1879 /* init original jump addresses which have been set during tcg_gen_code() */
1880 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1881 tb_reset_jump(tb, 0);
1883 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1884 tb_reset_jump(tb, 1);
1887 /* check next page if needed */
1888 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1889 phys_page2 = -1;
1890 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1891 phys_page2 = get_page_addr_code(env, virt_page2);
1894 * No explicit memory barrier is required -- tb_link_page() makes the
1895 * TB visible in a consistent state.
1897 existing_tb = tb_link_page(tb, phys_pc, phys_page2);
1898 /* if the TB already exists, discard what we just translated */
1899 if (unlikely(existing_tb != tb)) {
1900 uintptr_t orig_aligned = (uintptr_t)gen_code_buf;
1902 orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize);
1903 atomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned);
1904 tb_destroy(tb);
1905 return existing_tb;
1907 tcg_tb_insert(tb);
1908 return tb;
1912 * @p must be non-NULL.
1913 * user-mode: call with mmap_lock held.
1914 * !user-mode: call with all @pages locked.
1916 static void
1917 tb_invalidate_phys_page_range__locked(struct page_collection *pages,
1918 PageDesc *p, tb_page_addr_t start,
1919 tb_page_addr_t end,
1920 uintptr_t retaddr)
1922 TranslationBlock *tb;
1923 tb_page_addr_t tb_start, tb_end;
1924 int n;
1925 #ifdef TARGET_HAS_PRECISE_SMC
1926 CPUState *cpu = current_cpu;
1927 CPUArchState *env = NULL;
1928 bool current_tb_not_found = retaddr != 0;
1929 bool current_tb_modified = false;
1930 TranslationBlock *current_tb = NULL;
1931 target_ulong current_pc = 0;
1932 target_ulong current_cs_base = 0;
1933 uint32_t current_flags = 0;
1934 #endif /* TARGET_HAS_PRECISE_SMC */
1936 assert_page_locked(p);
1938 #if defined(TARGET_HAS_PRECISE_SMC)
1939 if (cpu != NULL) {
1940 env = cpu->env_ptr;
1942 #endif
1944 /* we remove all the TBs in the range [start, end[ */
1945 /* XXX: see if in some cases it could be faster to invalidate all
1946 the code */
1947 PAGE_FOR_EACH_TB(p, tb, n) {
1948 assert_page_locked(p);
1949 /* NOTE: this is subtle as a TB may span two physical pages */
1950 if (n == 0) {
1951 /* NOTE: tb_end may be after the end of the page, but
1952 it is not a problem */
1953 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1954 tb_end = tb_start + tb->size;
1955 } else {
1956 tb_start = tb->page_addr[1];
1957 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1959 if (!(tb_end <= start || tb_start >= end)) {
1960 #ifdef TARGET_HAS_PRECISE_SMC
1961 if (current_tb_not_found) {
1962 current_tb_not_found = false;
1963 /* now we have a real cpu fault */
1964 current_tb = tcg_tb_lookup(retaddr);
1966 if (current_tb == tb &&
1967 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
1969 * If we are modifying the current TB, we must stop
1970 * its execution. We could be more precise by checking
1971 * that the modification is after the current PC, but it
1972 * would require a specialized function to partially
1973 * restore the CPU state.
1975 current_tb_modified = true;
1976 cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
1977 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1978 &current_flags);
1980 #endif /* TARGET_HAS_PRECISE_SMC */
1981 tb_phys_invalidate__locked(tb);
1984 #if !defined(CONFIG_USER_ONLY)
1985 /* if no code remaining, no need to continue to use slow writes */
1986 if (!p->first_tb) {
1987 invalidate_page_bitmap(p);
1988 tlb_unprotect_code(start);
1990 #endif
1991 #ifdef TARGET_HAS_PRECISE_SMC
1992 if (current_tb_modified) {
1993 page_collection_unlock(pages);
1994 /* Force execution of one insn next time. */
1995 cpu->cflags_next_tb = 1 | curr_cflags();
1996 mmap_unlock();
1997 cpu_loop_exit_noexc(cpu);
1999 #endif
2003 * Invalidate all TBs which intersect with the target physical address range
2004 * [start;end[. NOTE: start and end must refer to the *same* physical page.
2005 * 'is_cpu_write_access' should be true if called from a real cpu write
2006 * access: the virtual CPU will exit the current TB if code is modified inside
2007 * this TB.
2009 * Called with mmap_lock held for user-mode emulation
2011 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
2013 struct page_collection *pages;
2014 PageDesc *p;
2016 assert_memory_lock();
2018 p = page_find(start >> TARGET_PAGE_BITS);
2019 if (p == NULL) {
2020 return;
2022 pages = page_collection_lock(start, end);
2023 tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
2024 page_collection_unlock(pages);
2028 * Invalidate all TBs which intersect with the target physical address range
2029 * [start;end[. NOTE: start and end may refer to *different* physical pages.
2030 * 'is_cpu_write_access' should be true if called from a real cpu write
2031 * access: the virtual CPU will exit the current TB if code is modified inside
2032 * this TB.
2034 * Called with mmap_lock held for user-mode emulation.
2036 #ifdef CONFIG_SOFTMMU
2037 void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
2038 #else
2039 void tb_invalidate_phys_range(target_ulong start, target_ulong end)
2040 #endif
2042 struct page_collection *pages;
2043 tb_page_addr_t next;
2045 assert_memory_lock();
2047 pages = page_collection_lock(start, end);
2048 for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
2049 start < end;
2050 start = next, next += TARGET_PAGE_SIZE) {
2051 PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
2052 tb_page_addr_t bound = MIN(next, end);
2054 if (pd == NULL) {
2055 continue;
2057 tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
2059 page_collection_unlock(pages);
2062 #ifdef CONFIG_SOFTMMU
2063 /* len must be <= 8 and start must be a multiple of len.
2064 * Called via softmmu_template.h when code areas are written to with
2065 * iothread mutex not held.
2067 * Call with all @pages in the range [@start, @start + len[ locked.
2069 void tb_invalidate_phys_page_fast(struct page_collection *pages,
2070 tb_page_addr_t start, int len,
2071 uintptr_t retaddr)
2073 PageDesc *p;
2075 assert_memory_lock();
2077 p = page_find(start >> TARGET_PAGE_BITS);
2078 if (!p) {
2079 return;
2082 assert_page_locked(p);
2083 if (!p->code_bitmap &&
2084 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
2085 build_page_bitmap(p);
2087 if (p->code_bitmap) {
2088 unsigned int nr;
2089 unsigned long b;
2091 nr = start & ~TARGET_PAGE_MASK;
2092 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
2093 if (b & ((1 << len) - 1)) {
2094 goto do_invalidate;
2096 } else {
2097 do_invalidate:
2098 tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
2099 retaddr);
2102 #else
2103 /* Called with mmap_lock held. If pc is not 0 then it indicates the
2104 * host PC of the faulting store instruction that caused this invalidate.
2105 * Returns true if the caller needs to abort execution of the current
2106 * TB (because it was modified by this store and the guest CPU has
2107 * precise-SMC semantics).
2109 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
2111 TranslationBlock *tb;
2112 PageDesc *p;
2113 int n;
2114 #ifdef TARGET_HAS_PRECISE_SMC
2115 TranslationBlock *current_tb = NULL;
2116 CPUState *cpu = current_cpu;
2117 CPUArchState *env = NULL;
2118 int current_tb_modified = 0;
2119 target_ulong current_pc = 0;
2120 target_ulong current_cs_base = 0;
2121 uint32_t current_flags = 0;
2122 #endif
2124 assert_memory_lock();
2126 addr &= TARGET_PAGE_MASK;
2127 p = page_find(addr >> TARGET_PAGE_BITS);
2128 if (!p) {
2129 return false;
2132 #ifdef TARGET_HAS_PRECISE_SMC
2133 if (p->first_tb && pc != 0) {
2134 current_tb = tcg_tb_lookup(pc);
2136 if (cpu != NULL) {
2137 env = cpu->env_ptr;
2139 #endif
2140 assert_page_locked(p);
2141 PAGE_FOR_EACH_TB(p, tb, n) {
2142 #ifdef TARGET_HAS_PRECISE_SMC
2143 if (current_tb == tb &&
2144 (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
2145 /* If we are modifying the current TB, we must stop
2146 its execution. We could be more precise by checking
2147 that the modification is after the current PC, but it
2148 would require a specialized function to partially
2149 restore the CPU state */
2151 current_tb_modified = 1;
2152 cpu_restore_state_from_tb(cpu, current_tb, pc, true);
2153 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
2154 &current_flags);
2156 #endif /* TARGET_HAS_PRECISE_SMC */
2157 tb_phys_invalidate(tb, addr);
2159 p->first_tb = (uintptr_t)NULL;
2160 #ifdef TARGET_HAS_PRECISE_SMC
2161 if (current_tb_modified) {
2162 /* Force execution of one insn next time. */
2163 cpu->cflags_next_tb = 1 | curr_cflags();
2164 return true;
2166 #endif
2168 return false;
2170 #endif
2172 /* user-mode: call with mmap_lock held */
2173 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
2175 TranslationBlock *tb;
2177 assert_memory_lock();
2179 tb = tcg_tb_lookup(retaddr);
2180 if (tb) {
2181 /* We can use retranslation to find the PC. */
2182 cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2183 tb_phys_invalidate(tb, -1);
2184 } else {
2185 /* The exception probably happened in a helper. The CPU state should
2186 have been saved before calling it. Fetch the PC from there. */
2187 CPUArchState *env = cpu->env_ptr;
2188 target_ulong pc, cs_base;
2189 tb_page_addr_t addr;
2190 uint32_t flags;
2192 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
2193 addr = get_page_addr_code(env, pc);
2194 if (addr != -1) {
2195 tb_invalidate_phys_range(addr, addr + 1);
2200 #ifndef CONFIG_USER_ONLY
2201 /* in deterministic execution mode, instructions doing device I/Os
2202 * must be at the end of the TB.
2204 * Called by softmmu_template.h, with iothread mutex not held.
2206 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
2208 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
2209 CPUArchState *env = cpu->env_ptr;
2210 #endif
2211 TranslationBlock *tb;
2212 uint32_t n;
2214 tb = tcg_tb_lookup(retaddr);
2215 if (!tb) {
2216 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
2217 (void *)retaddr);
2219 cpu_restore_state_from_tb(cpu, tb, retaddr, true);
2221 /* On MIPS and SH, delay slot instructions can only be restarted if
2222 they were already the first instruction in the TB. If this is not
2223 the first instruction in a TB then re-execute the preceding
2224 branch. */
2225 n = 1;
2226 #if defined(TARGET_MIPS)
2227 if ((env->hflags & MIPS_HFLAG_BMASK) != 0
2228 && env->active_tc.PC != tb->pc) {
2229 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
2230 cpu_neg(cpu)->icount_decr.u16.low++;
2231 env->hflags &= ~MIPS_HFLAG_BMASK;
2232 n = 2;
2234 #elif defined(TARGET_SH4)
2235 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
2236 && env->pc != tb->pc) {
2237 env->pc -= 2;
2238 cpu_neg(cpu)->icount_decr.u16.low++;
2239 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
2240 n = 2;
2242 #endif
2244 /* Generate a new TB executing the I/O insn. */
2245 cpu->cflags_next_tb = curr_cflags() | CF_LAST_IO | n;
2247 if (tb_cflags(tb) & CF_NOCACHE) {
2248 if (tb->orig_tb) {
2249 /* Invalidate original TB if this TB was generated in
2250 * cpu_exec_nocache() */
2251 tb_phys_invalidate(tb->orig_tb, -1);
2253 tcg_tb_remove(tb);
2254 tb_destroy(tb);
2257 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
2258 * the first in the TB) then we end up generating a whole new TB and
2259 * repeating the fault, which is horribly inefficient.
2260 * Better would be to execute just this insn uncached, or generate a
2261 * second new TB.
2263 cpu_loop_exit_noexc(cpu);
2266 static void tb_jmp_cache_clear_page(CPUState *cpu, target_ulong page_addr)
2268 unsigned int i, i0 = tb_jmp_cache_hash_page(page_addr);
2270 for (i = 0; i < TB_JMP_PAGE_SIZE; i++) {
2271 atomic_set(&cpu->tb_jmp_cache[i0 + i], NULL);
2275 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
2277 /* Discard jump cache entries for any tb which might potentially
2278 overlap the flushed page. */
2279 tb_jmp_cache_clear_page(cpu, addr - TARGET_PAGE_SIZE);
2280 tb_jmp_cache_clear_page(cpu, addr);
2283 static void print_qht_statistics(struct qht_stats hst)
2285 uint32_t hgram_opts;
2286 size_t hgram_bins;
2287 char *hgram;
2289 if (!hst.head_buckets) {
2290 return;
2292 qemu_printf("TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
2293 hst.used_head_buckets, hst.head_buckets,
2294 (double)hst.used_head_buckets / hst.head_buckets * 100);
2296 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
2297 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT;
2298 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
2299 hgram_opts |= QDIST_PR_NODECIMAL;
2301 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
2302 qemu_printf("TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
2303 qdist_avg(&hst.occupancy) * 100, hgram);
2304 g_free(hgram);
2306 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
2307 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
2308 if (hgram_bins > 10) {
2309 hgram_bins = 10;
2310 } else {
2311 hgram_bins = 0;
2312 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
2314 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
2315 qemu_printf("TB hash avg chain %0.3f buckets. Histogram: %s\n",
2316 qdist_avg(&hst.chain), hgram);
2317 g_free(hgram);
2320 struct tb_tree_stats {
2321 size_t nb_tbs;
2322 size_t host_size;
2323 size_t target_size;
2324 size_t max_target_size;
2325 size_t direct_jmp_count;
2326 size_t direct_jmp2_count;
2327 size_t cross_page;
2330 static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
2332 const TranslationBlock *tb = value;
2333 struct tb_tree_stats *tst = data;
2335 tst->nb_tbs++;
2336 tst->host_size += tb->tc.size;
2337 tst->target_size += tb->size;
2338 if (tb->size > tst->max_target_size) {
2339 tst->max_target_size = tb->size;
2341 if (tb->page_addr[1] != -1) {
2342 tst->cross_page++;
2344 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
2345 tst->direct_jmp_count++;
2346 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
2347 tst->direct_jmp2_count++;
2350 return false;
2353 void dump_exec_info(void)
2355 struct tb_tree_stats tst = {};
2356 struct qht_stats hst;
2357 size_t nb_tbs, flush_full, flush_part, flush_elide;
2359 tcg_tb_foreach(tb_tree_stats_iter, &tst);
2360 nb_tbs = tst.nb_tbs;
2361 /* XXX: avoid using doubles ? */
2362 qemu_printf("Translation buffer state:\n");
2364 * Report total code size including the padding and TB structs;
2365 * otherwise users might think "-tb-size" is not honoured.
2366 * For avg host size we use the precise numbers from tb_tree_stats though.
2368 qemu_printf("gen code size %zu/%zu\n",
2369 tcg_code_size(), tcg_code_capacity());
2370 qemu_printf("TB count %zu\n", nb_tbs);
2371 qemu_printf("TB avg target size %zu max=%zu bytes\n",
2372 nb_tbs ? tst.target_size / nb_tbs : 0,
2373 tst.max_target_size);
2374 qemu_printf("TB avg host size %zu bytes (expansion ratio: %0.1f)\n",
2375 nb_tbs ? tst.host_size / nb_tbs : 0,
2376 tst.target_size ? (double)tst.host_size / tst.target_size : 0);
2377 qemu_printf("cross page TB count %zu (%zu%%)\n", tst.cross_page,
2378 nb_tbs ? (tst.cross_page * 100) / nb_tbs : 0);
2379 qemu_printf("direct jump count %zu (%zu%%) (2 jumps=%zu %zu%%)\n",
2380 tst.direct_jmp_count,
2381 nb_tbs ? (tst.direct_jmp_count * 100) / nb_tbs : 0,
2382 tst.direct_jmp2_count,
2383 nb_tbs ? (tst.direct_jmp2_count * 100) / nb_tbs : 0);
2385 qht_statistics_init(&tb_ctx.htable, &hst);
2386 print_qht_statistics(hst);
2387 qht_statistics_destroy(&hst);
2389 qemu_printf("\nStatistics:\n");
2390 qemu_printf("TB flush count %u\n",
2391 atomic_read(&tb_ctx.tb_flush_count));
2392 qemu_printf("TB invalidate count %zu\n",
2393 tcg_tb_phys_invalidate_count());
2395 tlb_flush_counts(&flush_full, &flush_part, &flush_elide);
2396 qemu_printf("TLB full flushes %zu\n", flush_full);
2397 qemu_printf("TLB partial flushes %zu\n", flush_part);
2398 qemu_printf("TLB elided flushes %zu\n", flush_elide);
2399 tcg_dump_info();
2402 void dump_opcount_info(void)
2404 tcg_dump_op_count();
2407 #else /* CONFIG_USER_ONLY */
2409 void cpu_interrupt(CPUState *cpu, int mask)
2411 g_assert(qemu_mutex_iothread_locked());
2412 cpu->interrupt_request |= mask;
2413 atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
2417 * Walks guest process memory "regions" one by one
2418 * and calls callback function 'fn' for each region.
2420 struct walk_memory_regions_data {
2421 walk_memory_regions_fn fn;
2422 void *priv;
2423 target_ulong start;
2424 int prot;
2427 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
2428 target_ulong end, int new_prot)
2430 if (data->start != -1u) {
2431 int rc = data->fn(data->priv, data->start, end, data->prot);
2432 if (rc != 0) {
2433 return rc;
2437 data->start = (new_prot ? end : -1u);
2438 data->prot = new_prot;
2440 return 0;
2443 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
2444 target_ulong base, int level, void **lp)
2446 target_ulong pa;
2447 int i, rc;
2449 if (*lp == NULL) {
2450 return walk_memory_regions_end(data, base, 0);
2453 if (level == 0) {
2454 PageDesc *pd = *lp;
2456 for (i = 0; i < V_L2_SIZE; ++i) {
2457 int prot = pd[i].flags;
2459 pa = base | (i << TARGET_PAGE_BITS);
2460 if (prot != data->prot) {
2461 rc = walk_memory_regions_end(data, pa, prot);
2462 if (rc != 0) {
2463 return rc;
2467 } else {
2468 void **pp = *lp;
2470 for (i = 0; i < V_L2_SIZE; ++i) {
2471 pa = base | ((target_ulong)i <<
2472 (TARGET_PAGE_BITS + V_L2_BITS * level));
2473 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
2474 if (rc != 0) {
2475 return rc;
2480 return 0;
2483 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
2485 struct walk_memory_regions_data data;
2486 uintptr_t i, l1_sz = v_l1_size;
2488 data.fn = fn;
2489 data.priv = priv;
2490 data.start = -1u;
2491 data.prot = 0;
2493 for (i = 0; i < l1_sz; i++) {
2494 target_ulong base = i << (v_l1_shift + TARGET_PAGE_BITS);
2495 int rc = walk_memory_regions_1(&data, base, v_l2_levels, l1_map + i);
2496 if (rc != 0) {
2497 return rc;
2501 return walk_memory_regions_end(&data, 0, 0);
2504 static int dump_region(void *priv, target_ulong start,
2505 target_ulong end, unsigned long prot)
2507 FILE *f = (FILE *)priv;
2509 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
2510 " "TARGET_FMT_lx" %c%c%c\n",
2511 start, end, end - start,
2512 ((prot & PAGE_READ) ? 'r' : '-'),
2513 ((prot & PAGE_WRITE) ? 'w' : '-'),
2514 ((prot & PAGE_EXEC) ? 'x' : '-'));
2516 return 0;
2519 /* dump memory mappings */
2520 void page_dump(FILE *f)
2522 const int length = sizeof(target_ulong) * 2;
2523 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
2524 length, "start", length, "end", length, "size", "prot");
2525 walk_memory_regions(f, dump_region);
2528 int page_get_flags(target_ulong address)
2530 PageDesc *p;
2532 p = page_find(address >> TARGET_PAGE_BITS);
2533 if (!p) {
2534 return 0;
2536 return p->flags;
2539 /* Modify the flags of a page and invalidate the code if necessary.
2540 The flag PAGE_WRITE_ORG is positioned automatically depending
2541 on PAGE_WRITE. The mmap_lock should already be held. */
2542 void page_set_flags(target_ulong start, target_ulong end, int flags)
2544 target_ulong addr, len;
2546 /* This function should never be called with addresses outside the
2547 guest address space. If this assert fires, it probably indicates
2548 a missing call to h2g_valid. */
2549 assert(end - 1 <= GUEST_ADDR_MAX);
2550 assert(start < end);
2551 assert_memory_lock();
2553 start = start & TARGET_PAGE_MASK;
2554 end = TARGET_PAGE_ALIGN(end);
2556 if (flags & PAGE_WRITE) {
2557 flags |= PAGE_WRITE_ORG;
2560 for (addr = start, len = end - start;
2561 len != 0;
2562 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2563 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2565 /* If the write protection bit is set, then we invalidate
2566 the code inside. */
2567 if (!(p->flags & PAGE_WRITE) &&
2568 (flags & PAGE_WRITE) &&
2569 p->first_tb) {
2570 tb_invalidate_phys_page(addr, 0);
2572 p->flags = flags;
2576 int page_check_range(target_ulong start, target_ulong len, int flags)
2578 PageDesc *p;
2579 target_ulong end;
2580 target_ulong addr;
2582 /* This function should never be called with addresses outside the
2583 guest address space. If this assert fires, it probably indicates
2584 a missing call to h2g_valid. */
2585 if (TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS) {
2586 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
2589 if (len == 0) {
2590 return 0;
2592 if (start + len - 1 < start) {
2593 /* We've wrapped around. */
2594 return -1;
2597 /* must do before we loose bits in the next step */
2598 end = TARGET_PAGE_ALIGN(start + len);
2599 start = start & TARGET_PAGE_MASK;
2601 for (addr = start, len = end - start;
2602 len != 0;
2603 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2604 p = page_find(addr >> TARGET_PAGE_BITS);
2605 if (!p) {
2606 return -1;
2608 if (!(p->flags & PAGE_VALID)) {
2609 return -1;
2612 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
2613 return -1;
2615 if (flags & PAGE_WRITE) {
2616 if (!(p->flags & PAGE_WRITE_ORG)) {
2617 return -1;
2619 /* unprotect the page if it was put read-only because it
2620 contains translated code */
2621 if (!(p->flags & PAGE_WRITE)) {
2622 if (!page_unprotect(addr, 0)) {
2623 return -1;
2628 return 0;
2631 /* called from signal handler: invalidate the code and unprotect the
2632 * page. Return 0 if the fault was not handled, 1 if it was handled,
2633 * and 2 if it was handled but the caller must cause the TB to be
2634 * immediately exited. (We can only return 2 if the 'pc' argument is
2635 * non-zero.)
2637 int page_unprotect(target_ulong address, uintptr_t pc)
2639 unsigned int prot;
2640 bool current_tb_invalidated;
2641 PageDesc *p;
2642 target_ulong host_start, host_end, addr;
2644 /* Technically this isn't safe inside a signal handler. However we
2645 know this only ever happens in a synchronous SEGV handler, so in
2646 practice it seems to be ok. */
2647 mmap_lock();
2649 p = page_find(address >> TARGET_PAGE_BITS);
2650 if (!p) {
2651 mmap_unlock();
2652 return 0;
2655 /* if the page was really writable, then we change its
2656 protection back to writable */
2657 if (p->flags & PAGE_WRITE_ORG) {
2658 current_tb_invalidated = false;
2659 if (p->flags & PAGE_WRITE) {
2660 /* If the page is actually marked WRITE then assume this is because
2661 * this thread raced with another one which got here first and
2662 * set the page to PAGE_WRITE and did the TB invalidate for us.
2664 #ifdef TARGET_HAS_PRECISE_SMC
2665 TranslationBlock *current_tb = tcg_tb_lookup(pc);
2666 if (current_tb) {
2667 current_tb_invalidated = tb_cflags(current_tb) & CF_INVALID;
2669 #endif
2670 } else {
2671 host_start = address & qemu_host_page_mask;
2672 host_end = host_start + qemu_host_page_size;
2674 prot = 0;
2675 for (addr = host_start; addr < host_end; addr += TARGET_PAGE_SIZE) {
2676 p = page_find(addr >> TARGET_PAGE_BITS);
2677 p->flags |= PAGE_WRITE;
2678 prot |= p->flags;
2680 /* and since the content will be modified, we must invalidate
2681 the corresponding translated code. */
2682 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
2683 #ifdef CONFIG_USER_ONLY
2684 if (DEBUG_TB_CHECK_GATE) {
2685 tb_invalidate_check(addr);
2687 #endif
2689 mprotect((void *)g2h(host_start), qemu_host_page_size,
2690 prot & PAGE_BITS);
2692 mmap_unlock();
2693 /* If current TB was invalidated return to main loop */
2694 return current_tb_invalidated ? 2 : 1;
2696 mmap_unlock();
2697 return 0;
2699 #endif /* CONFIG_USER_ONLY */
2701 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2702 void tcg_flush_softmmu_tlb(CPUState *cs)
2704 #ifdef CONFIG_SOFTMMU
2705 tlb_flush(cs);
2706 #endif