4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "qemu/osdep.h"
24 #include "qemu-common.h"
25 #define NO_CPU_IO_DEFS
28 #include "disas/disas.h"
29 #include "exec/exec-all.h"
31 #if defined(CONFIG_USER_ONLY)
33 #if defined(TARGET_X86_64)
36 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
37 #include <sys/param.h>
38 #if __FreeBSD_version >= 700104
39 #define HAVE_KINFO_GETVMMAP
40 #define sigqueue sigqueue_freebsd /* avoid redefinition */
42 #include <machine/profile.h>
51 #include "exec/address-spaces.h"
54 #include "exec/cputlb.h"
55 #include "exec/tb-hash.h"
56 #include "translate-all.h"
57 #include "qemu/bitmap.h"
58 #include "qemu/timer.h"
61 //#define DEBUG_TB_INVALIDATE
63 /* make various TB consistency checks */
64 //#define DEBUG_TB_CHECK
66 #if !defined(CONFIG_USER_ONLY)
67 /* TB consistency checks only implemented for usermode emulation. */
71 #define SMC_BITMAP_USE_THRESHOLD 10
73 typedef struct PageDesc
{
74 /* list of TBs intersecting this ram page */
75 TranslationBlock
*first_tb
;
77 /* in order to optimize self modifying code, we count the number
78 of lookups we do to a given page to use a bitmap */
79 unsigned int code_write_count
;
80 unsigned long *code_bitmap
;
86 /* In system mode we want L1_MAP to be based on ram offsets,
87 while in user mode we want it to be based on virtual addresses. */
88 #if !defined(CONFIG_USER_ONLY)
89 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
90 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
92 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
95 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
98 /* Size of the L2 (and L3, etc) page tables. */
100 #define V_L2_SIZE (1 << V_L2_BITS)
102 /* The bits remaining after N lower levels of page tables. */
103 #define V_L1_BITS_REM \
104 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
106 #if V_L1_BITS_REM < 4
107 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
109 #define V_L1_BITS V_L1_BITS_REM
112 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
114 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
116 uintptr_t qemu_host_page_size
;
117 intptr_t qemu_host_page_mask
;
119 /* The bottom level has pointers to PageDesc */
120 static void *l1_map
[V_L1_SIZE
];
122 /* code generation context */
125 /* translation block context */
126 #ifdef CONFIG_USER_ONLY
127 __thread
int have_tb_lock
;
132 #ifdef CONFIG_USER_ONLY
133 assert(!have_tb_lock
);
134 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
141 #ifdef CONFIG_USER_ONLY
142 assert(have_tb_lock
);
144 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
148 void tb_lock_reset(void)
150 #ifdef CONFIG_USER_ONLY
152 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
158 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
160 void cpu_gen_init(void)
162 tcg_context_init(&tcg_ctx
);
165 /* Encode VAL as a signed leb128 sequence at P.
166 Return P incremented past the encoded value. */
167 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
174 more
= !((val
== 0 && (byte
& 0x40) == 0)
175 || (val
== -1 && (byte
& 0x40) != 0));
185 /* Decode a signed leb128 sequence at *PP; increment *PP past the
186 decoded value. Return the decoded value. */
187 static target_long
decode_sleb128(uint8_t **pp
)
195 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
197 } while (byte
& 0x80);
198 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
199 val
|= -(target_ulong
)1 << shift
;
206 /* Encode the data collected about the instructions while compiling TB.
207 Place the data at BLOCK, and return the number of bytes consumed.
209 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
210 which come from the target's insn_start data, followed by a uintptr_t
211 which comes from the host pc of the end of the code implementing the insn.
213 Each line of the table is encoded as sleb128 deltas from the previous
214 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
215 That is, the first column is seeded with the guest pc, the last column
216 with the host pc, and the middle columns with zeros. */
218 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
220 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
224 tb
->tc_search
= block
;
226 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
229 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
231 prev
= (j
== 0 ? tb
->pc
: 0);
233 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
235 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
237 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
238 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
240 /* Test for (pending) buffer overflow. The assumption is that any
241 one row beginning below the high water mark cannot overrun
242 the buffer completely. Thus we can test for overflow after
243 encoding a row without having to check during encoding. */
244 if (unlikely(p
> highwater
)) {
252 /* The cpu state corresponding to 'searched_pc' is restored. */
253 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
254 uintptr_t searched_pc
)
256 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
257 uintptr_t host_pc
= (uintptr_t)tb
->tc_ptr
;
258 CPUArchState
*env
= cpu
->env_ptr
;
259 uint8_t *p
= tb
->tc_search
;
260 int i
, j
, num_insns
= tb
->icount
;
261 #ifdef CONFIG_PROFILER
262 int64_t ti
= profile_getclock();
265 if (searched_pc
< host_pc
) {
269 /* Reconstruct the stored insn data while looking for the point at
270 which the end of the insn exceeds the searched_pc. */
271 for (i
= 0; i
< num_insns
; ++i
) {
272 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
273 data
[j
] += decode_sleb128(&p
);
275 host_pc
+= decode_sleb128(&p
);
276 if (host_pc
> searched_pc
) {
283 if (tb
->cflags
& CF_USE_ICOUNT
) {
285 /* Reset the cycle counter to the start of the block. */
286 cpu
->icount_decr
.u16
.low
+= num_insns
;
287 /* Clear the IO flag. */
290 cpu
->icount_decr
.u16
.low
-= i
;
291 restore_state_to_opc(env
, tb
, data
);
293 #ifdef CONFIG_PROFILER
294 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
295 tcg_ctx
.restore_count
++;
300 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
302 TranslationBlock
*tb
;
304 tb
= tb_find_pc(retaddr
);
306 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
307 if (tb
->cflags
& CF_NOCACHE
) {
308 /* one-shot translation, invalidate it immediately */
309 tb_phys_invalidate(tb
, -1);
317 void page_size_init(void)
319 /* NOTE: we can always suppose that qemu_host_page_size >=
321 qemu_real_host_page_size
= getpagesize();
322 qemu_real_host_page_mask
= -(intptr_t)qemu_real_host_page_size
;
323 if (qemu_host_page_size
== 0) {
324 qemu_host_page_size
= qemu_real_host_page_size
;
326 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
327 qemu_host_page_size
= TARGET_PAGE_SIZE
;
329 qemu_host_page_mask
= -(intptr_t)qemu_host_page_size
;
332 static void page_init(void)
335 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
337 #ifdef HAVE_KINFO_GETVMMAP
338 struct kinfo_vmentry
*freep
;
341 freep
= kinfo_getvmmap(getpid(), &cnt
);
344 for (i
= 0; i
< cnt
; i
++) {
345 unsigned long startaddr
, endaddr
;
347 startaddr
= freep
[i
].kve_start
;
348 endaddr
= freep
[i
].kve_end
;
349 if (h2g_valid(startaddr
)) {
350 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
352 if (h2g_valid(endaddr
)) {
353 endaddr
= h2g(endaddr
);
354 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
356 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
358 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
369 last_brk
= (unsigned long)sbrk(0);
371 f
= fopen("/compat/linux/proc/self/maps", "r");
376 unsigned long startaddr
, endaddr
;
379 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
381 if (n
== 2 && h2g_valid(startaddr
)) {
382 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
384 if (h2g_valid(endaddr
)) {
385 endaddr
= h2g(endaddr
);
389 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
402 * Called with mmap_lock held for user-mode emulation.
404 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
410 /* Level 1. Always allocated. */
411 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
414 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
415 void **p
= atomic_rcu_read(lp
);
421 p
= g_new0(void *, V_L2_SIZE
);
422 atomic_rcu_set(lp
, p
);
425 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
428 pd
= atomic_rcu_read(lp
);
433 pd
= g_new0(PageDesc
, V_L2_SIZE
);
434 atomic_rcu_set(lp
, pd
);
437 return pd
+ (index
& (V_L2_SIZE
- 1));
440 static inline PageDesc
*page_find(tb_page_addr_t index
)
442 return page_find_alloc(index
, 0);
445 #if defined(CONFIG_USER_ONLY)
446 /* Currently it is not recommended to allocate big chunks of data in
447 user mode. It will change when a dedicated libc will be used. */
448 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
449 region in which the guest needs to run. Revisit this. */
450 #define USE_STATIC_CODE_GEN_BUFFER
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__powerpc64__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
466 #elif defined(__powerpc__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
468 #elif defined(__aarch64__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
470 #elif defined(__arm__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
472 #elif defined(__s390x__)
473 /* We have a +- 4GB range on the branches; leave some slop. */
474 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
475 #elif defined(__mips__)
476 /* We have a 256MB branch region, but leave room to make sure the
477 main executable is also within that region. */
478 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
480 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
485 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
486 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
487 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
489 static inline size_t size_code_gen_buffer(size_t tb_size
)
491 /* Size the buffer. */
493 #ifdef USE_STATIC_CODE_GEN_BUFFER
494 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
496 /* ??? Needs adjustments. */
497 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
498 static buffer, we could size this on RESERVED_VA, on the text
499 segment size of the executable, or continue to use the default. */
500 tb_size
= (unsigned long)(ram_size
/ 4);
503 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
504 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
506 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
507 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
513 /* In order to use J and JAL within the code_gen_buffer, we require
514 that the buffer not cross a 256MB boundary. */
515 static inline bool cross_256mb(void *addr
, size_t size
)
517 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
520 /* We weren't able to allocate a buffer without crossing that boundary,
521 so make do with the larger portion of the buffer that doesn't cross.
522 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
523 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
525 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
526 size_t size2
= buf1
+ size1
- buf2
;
534 tcg_ctx
.code_gen_buffer_size
= size1
;
539 #ifdef USE_STATIC_CODE_GEN_BUFFER
540 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
541 __attribute__((aligned(CODE_GEN_ALIGN
)));
544 static inline void do_protect(void *addr
, long size
, int prot
)
547 VirtualProtect(addr
, size
, prot
, &old_protect
);
550 static inline void map_exec(void *addr
, long size
)
552 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
555 static inline void map_none(void *addr
, long size
)
557 do_protect(addr
, size
, PAGE_NOACCESS
);
560 static inline void do_protect(void *addr
, long size
, int prot
)
562 uintptr_t start
, end
;
564 start
= (uintptr_t)addr
;
565 start
&= qemu_real_host_page_mask
;
567 end
= (uintptr_t)addr
+ size
;
568 end
= ROUND_UP(end
, qemu_real_host_page_size
);
570 mprotect((void *)start
, end
- start
, prot
);
573 static inline void map_exec(void *addr
, long size
)
575 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
578 static inline void map_none(void *addr
, long size
)
580 do_protect(addr
, size
, PROT_NONE
);
584 static inline void *alloc_code_gen_buffer(void)
586 void *buf
= static_code_gen_buffer
;
587 size_t full_size
, size
;
589 /* The size of the buffer, rounded down to end on a page boundary. */
590 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
591 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
593 /* Reserve a guard page. */
594 size
= full_size
- qemu_real_host_page_size
;
596 /* Honor a command-line option limiting the size of the buffer. */
597 if (size
> tcg_ctx
.code_gen_buffer_size
) {
598 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
599 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
601 tcg_ctx
.code_gen_buffer_size
= size
;
604 if (cross_256mb(buf
, size
)) {
605 buf
= split_cross_256mb(buf
, size
);
606 size
= tcg_ctx
.code_gen_buffer_size
;
611 map_none(buf
+ size
, qemu_real_host_page_size
);
612 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
616 #elif defined(_WIN32)
617 static inline void *alloc_code_gen_buffer(void)
619 size_t size
= tcg_ctx
.code_gen_buffer_size
;
622 /* Perform the allocation in two steps, so that the guard page
623 is reserved but uncommitted. */
624 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
625 MEM_RESERVE
, PAGE_NOACCESS
);
627 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
628 assert(buf1
== buf2
);
634 static inline void *alloc_code_gen_buffer(void)
636 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
638 size_t size
= tcg_ctx
.code_gen_buffer_size
;
641 /* Constrain the position of the buffer based on the host cpu.
642 Note that these addresses are chosen in concert with the
643 addresses assigned in the relevant linker script file. */
644 # if defined(__PIE__) || defined(__PIC__)
645 /* Don't bother setting a preferred location if we're building
646 a position-independent executable. We're more likely to get
647 an address near the main executable if we let the kernel
648 choose the address. */
649 # elif defined(__x86_64__) && defined(MAP_32BIT)
650 /* Force the memory down into low memory with the executable.
651 Leave the choice of exact location with the kernel. */
653 /* Cannot expect to map more than 800MB in low memory. */
654 if (size
> 800u * 1024 * 1024) {
655 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
657 # elif defined(__sparc__)
658 start
= 0x40000000ul
;
659 # elif defined(__s390x__)
660 start
= 0x90000000ul
;
661 # elif defined(__mips__)
662 # if _MIPS_SIM == _ABI64
663 start
= 0x128000000ul
;
665 start
= 0x08000000ul
;
669 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
670 PROT_NONE
, flags
, -1, 0);
671 if (buf
== MAP_FAILED
) {
676 if (cross_256mb(buf
, size
)) {
677 /* Try again, with the original still mapped, to avoid re-acquiring
678 that 256mb crossing. This time don't specify an address. */
680 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
681 PROT_NONE
, flags
, -1, 0);
682 switch (buf2
!= MAP_FAILED
) {
684 if (!cross_256mb(buf2
, size
)) {
685 /* Success! Use the new buffer. */
686 munmap(buf
, size
+ qemu_real_host_page_size
);
689 /* Failure. Work with what we had. */
690 munmap(buf2
, size
+ qemu_real_host_page_size
);
693 /* Split the original buffer. Free the smaller half. */
694 buf2
= split_cross_256mb(buf
, size
);
695 size2
= tcg_ctx
.code_gen_buffer_size
;
697 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
699 munmap(buf
, size
- size2
);
708 /* Make the final buffer accessible. The guard page at the end
709 will remain inaccessible with PROT_NONE. */
710 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
712 /* Request large pages for the buffer. */
713 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
717 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
719 static inline void code_gen_alloc(size_t tb_size
)
721 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
722 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
723 if (tcg_ctx
.code_gen_buffer
== NULL
) {
724 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
728 /* Estimate a good size for the number of TBs we can support. We
729 still haven't deducted the prologue from the buffer size here,
730 but that's minimal and won't affect the estimate much. */
731 tcg_ctx
.code_gen_max_blocks
732 = tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
733 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
, tcg_ctx
.code_gen_max_blocks
);
735 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
738 static void tb_htable_init(void)
740 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
742 qht_init(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
, mode
);
745 /* Must be called before using the QEMU cpus. 'tb_size' is the size
746 (in bytes) allocated to the translation buffer. Zero means default
748 void tcg_exec_init(uintptr_t tb_size
)
753 code_gen_alloc(tb_size
);
754 #if defined(CONFIG_SOFTMMU)
755 /* There's no guest base to take into account, so go ahead and
756 initialize the prologue now. */
757 tcg_prologue_init(&tcg_ctx
);
761 bool tcg_enabled(void)
763 return tcg_ctx
.code_gen_buffer
!= NULL
;
766 /* Allocate a new translation block. Flush the translation buffer if
767 too many translation blocks or too much generated code. */
768 static TranslationBlock
*tb_alloc(target_ulong pc
)
770 TranslationBlock
*tb
;
772 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
) {
775 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
781 void tb_free(TranslationBlock
*tb
)
783 /* In practice this is mostly used for single use temporary TB
784 Ignore the hard cases and just back up if this TB happens to
785 be the last one generated. */
786 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
787 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
788 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
789 tcg_ctx
.tb_ctx
.nb_tbs
--;
793 static inline void invalidate_page_bitmap(PageDesc
*p
)
795 #ifdef CONFIG_SOFTMMU
796 g_free(p
->code_bitmap
);
797 p
->code_bitmap
= NULL
;
798 p
->code_write_count
= 0;
802 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
803 static void page_flush_tb_1(int level
, void **lp
)
813 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
814 pd
[i
].first_tb
= NULL
;
815 invalidate_page_bitmap(pd
+ i
);
820 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
821 page_flush_tb_1(level
- 1, pp
+ i
);
826 static void page_flush_tb(void)
830 for (i
= 0; i
< V_L1_SIZE
; i
++) {
831 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
835 /* flush all the translation blocks */
836 /* XXX: tb_flush is currently not thread safe */
837 void tb_flush(CPUState
*cpu
)
839 #if defined(DEBUG_FLUSH)
840 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
841 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
842 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
843 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
844 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
846 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
847 > tcg_ctx
.code_gen_buffer_size
) {
848 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
850 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
853 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
854 cpu
->tb_flushed
= true;
857 qht_reset_size(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
860 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
861 /* XXX: flush processor icache at this point if cache flush is
863 tcg_ctx
.tb_ctx
.tb_flush_count
++;
866 #ifdef DEBUG_TB_CHECK
869 do_tb_invalidate_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
871 TranslationBlock
*tb
= p
;
872 target_ulong addr
= *(target_ulong
*)userp
;
874 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
875 printf("ERROR invalidate: address=" TARGET_FMT_lx
876 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
880 static void tb_invalidate_check(target_ulong address
)
882 address
&= TARGET_PAGE_MASK
;
883 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
887 do_tb_page_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
889 TranslationBlock
*tb
= p
;
892 flags1
= page_get_flags(tb
->pc
);
893 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
894 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
895 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
896 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
900 /* verify that all the pages have correct rights for code */
901 static void tb_page_check(void)
903 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_page_check
, NULL
);
908 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
910 TranslationBlock
*tb1
;
915 n1
= (uintptr_t)tb1
& 3;
916 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
918 *ptb
= tb1
->page_next
[n1
];
921 ptb
= &tb1
->page_next
[n1
];
925 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
926 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
928 TranslationBlock
*tb1
;
932 ptb
= &tb
->jmp_list_next
[n
];
934 /* find tb(n) in circular list */
938 tb1
= (TranslationBlock
*)(ntb
& ~3);
939 if (n1
== n
&& tb1
== tb
) {
943 ptb
= &tb1
->jmp_list_first
;
945 ptb
= &tb1
->jmp_list_next
[n1
];
948 /* now we can suppress tb(n) from the list */
949 *ptb
= tb
->jmp_list_next
[n
];
951 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
955 /* reset the jump entry 'n' of a TB so that it is not chained to
957 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
959 uintptr_t addr
= (uintptr_t)(tb
->tc_ptr
+ tb
->jmp_reset_offset
[n
]);
960 tb_set_jmp_target(tb
, n
, addr
);
963 /* remove any jumps to the TB */
964 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
966 TranslationBlock
*tb1
;
970 ptb
= &tb
->jmp_list_first
;
974 tb1
= (TranslationBlock
*)(ntb
& ~3);
978 tb_reset_jump(tb1
, n1
);
979 *ptb
= tb1
->jmp_list_next
[n1
];
980 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
984 /* invalidate one TB */
985 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
990 tb_page_addr_t phys_pc
;
992 /* remove the TB from the hash list */
993 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
994 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
995 qht_remove(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
997 /* remove the TB from the page list */
998 if (tb
->page_addr
[0] != page_addr
) {
999 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1000 tb_page_remove(&p
->first_tb
, tb
);
1001 invalidate_page_bitmap(p
);
1003 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1004 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1005 tb_page_remove(&p
->first_tb
, tb
);
1006 invalidate_page_bitmap(p
);
1009 /* remove the TB from the hash list */
1010 h
= tb_jmp_cache_hash_func(tb
->pc
);
1012 if (cpu
->tb_jmp_cache
[h
] == tb
) {
1013 cpu
->tb_jmp_cache
[h
] = NULL
;
1017 /* suppress this TB from the two jump lists */
1018 tb_remove_from_jmp_list(tb
, 0);
1019 tb_remove_from_jmp_list(tb
, 1);
1021 /* suppress any remaining jumps to this TB */
1024 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1027 #ifdef CONFIG_SOFTMMU
1028 static void build_page_bitmap(PageDesc
*p
)
1030 int n
, tb_start
, tb_end
;
1031 TranslationBlock
*tb
;
1033 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1036 while (tb
!= NULL
) {
1037 n
= (uintptr_t)tb
& 3;
1038 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1039 /* NOTE: this is subtle as a TB may span two physical pages */
1041 /* NOTE: tb_end may be after the end of the page, but
1042 it is not a problem */
1043 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1044 tb_end
= tb_start
+ tb
->size
;
1045 if (tb_end
> TARGET_PAGE_SIZE
) {
1046 tb_end
= TARGET_PAGE_SIZE
;
1050 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1052 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1053 tb
= tb
->page_next
[n
];
1058 /* add the tb in the target page and protect it if necessary
1060 * Called with mmap_lock held for user-mode emulation.
1062 static inline void tb_alloc_page(TranslationBlock
*tb
,
1063 unsigned int n
, tb_page_addr_t page_addr
)
1066 #ifndef CONFIG_USER_ONLY
1067 bool page_already_protected
;
1070 tb
->page_addr
[n
] = page_addr
;
1071 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1072 tb
->page_next
[n
] = p
->first_tb
;
1073 #ifndef CONFIG_USER_ONLY
1074 page_already_protected
= p
->first_tb
!= NULL
;
1076 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1077 invalidate_page_bitmap(p
);
1079 #if defined(CONFIG_USER_ONLY)
1080 if (p
->flags
& PAGE_WRITE
) {
1085 /* force the host page as non writable (writes will have a
1086 page fault + mprotect overhead) */
1087 page_addr
&= qemu_host_page_mask
;
1089 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1090 addr
+= TARGET_PAGE_SIZE
) {
1092 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1097 p2
->flags
&= ~PAGE_WRITE
;
1099 mprotect(g2h(page_addr
), qemu_host_page_size
,
1100 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1101 #ifdef DEBUG_TB_INVALIDATE
1102 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1107 /* if some code is already present, then the pages are already
1108 protected. So we handle the case where only the first TB is
1109 allocated in a physical page */
1110 if (!page_already_protected
) {
1111 tlb_protect_code(page_addr
);
1116 /* add a new TB and link it to the physical page tables. phys_page2 is
1117 * (-1) to indicate that only one page contains the TB.
1119 * Called with mmap_lock held for user-mode emulation.
1121 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1122 tb_page_addr_t phys_page2
)
1126 /* add in the hash table */
1127 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
);
1128 qht_insert(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1130 /* add in the page list */
1131 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1132 if (phys_page2
!= -1) {
1133 tb_alloc_page(tb
, 1, phys_page2
);
1135 tb
->page_addr
[1] = -1;
1138 #ifdef DEBUG_TB_CHECK
1143 /* Called with mmap_lock held for user mode emulation. */
1144 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1145 target_ulong pc
, target_ulong cs_base
,
1146 uint32_t flags
, int cflags
)
1148 CPUArchState
*env
= cpu
->env_ptr
;
1149 TranslationBlock
*tb
;
1150 tb_page_addr_t phys_pc
, phys_page2
;
1151 target_ulong virt_page2
;
1152 tcg_insn_unit
*gen_code_buf
;
1153 int gen_code_size
, search_size
;
1154 #ifdef CONFIG_PROFILER
1158 phys_pc
= get_page_addr_code(env
, pc
);
1159 if (use_icount
&& !(cflags
& CF_IGNORE_ICOUNT
)) {
1160 cflags
|= CF_USE_ICOUNT
;
1164 if (unlikely(!tb
)) {
1166 /* flush must be done */
1168 /* cannot fail at this point */
1173 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1174 tb
->tc_ptr
= gen_code_buf
;
1175 tb
->cs_base
= cs_base
;
1177 tb
->cflags
= cflags
;
1179 #ifdef CONFIG_PROFILER
1180 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1182 ti
= profile_getclock();
1185 tcg_func_start(&tcg_ctx
);
1187 tcg_ctx
.cpu
= ENV_GET_CPU(env
);
1188 gen_intermediate_code(env
, tb
);
1191 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
1193 /* generate machine code */
1194 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1195 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1196 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1197 #ifdef USE_DIRECT_JUMP
1198 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_insn_offset
;
1199 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1201 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1202 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_addr
;
1205 #ifdef CONFIG_PROFILER
1207 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1208 tcg_ctx
.code_time
-= profile_getclock();
1211 /* ??? Overflow could be handled better here. In particular, we
1212 don't need to re-do gen_intermediate_code, nor should we re-do
1213 the tcg optimization currently hidden inside tcg_gen_code. All
1214 that should be required is to flush the TBs, allocate a new TB,
1215 re-initialize it per above, and re-do the actual code generation. */
1216 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1217 if (unlikely(gen_code_size
< 0)) {
1218 goto buffer_overflow
;
1220 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1221 if (unlikely(search_size
< 0)) {
1222 goto buffer_overflow
;
1225 #ifdef CONFIG_PROFILER
1226 tcg_ctx
.code_time
+= profile_getclock();
1227 tcg_ctx
.code_in_len
+= tb
->size
;
1228 tcg_ctx
.code_out_len
+= gen_code_size
;
1229 tcg_ctx
.search_out_len
+= search_size
;
1233 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1234 qemu_log_in_addr_range(tb
->pc
)) {
1235 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1236 log_disas(tb
->tc_ptr
, gen_code_size
);
1242 tcg_ctx
.code_gen_ptr
= (void *)
1243 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1246 #if defined(CONFIG_USER_ONLY) && defined(TARGET_X86_64)
1247 /* if we are doing vsyscall don't link the page as it lies in high memory
1248 and tb_alloc_page will abort due to page_l1_map returning NULL */
1249 if (unlikely(phys_pc
>= TARGET_VSYSCALL_START
1250 && phys_pc
< TARGET_VSYSCALL_END
))
1254 /* init jump list */
1255 assert(((uintptr_t)tb
& 3) == 0);
1256 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1257 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1258 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1260 /* init original jump addresses wich has been set during tcg_gen_code() */
1261 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1262 tb_reset_jump(tb
, 0);
1264 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1265 tb_reset_jump(tb
, 1);
1268 /* check next page if needed */
1269 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1271 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1272 phys_page2
= get_page_addr_code(env
, virt_page2
);
1274 /* As long as consistency of the TB stuff is provided by tb_lock in user
1275 * mode and is implicit in single-threaded softmmu emulation, no explicit
1276 * memory barrier is required before tb_link_page() makes the TB visible
1277 * through the physical hash table and physical page list.
1279 tb_link_page(tb
, phys_pc
, phys_page2
);
1284 * Invalidate all TBs which intersect with the target physical address range
1285 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1286 * 'is_cpu_write_access' should be true if called from a real cpu write
1287 * access: the virtual CPU will exit the current TB if code is modified inside
1290 * Called with mmap_lock held for user-mode emulation
1292 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1294 while (start
< end
) {
1295 tb_invalidate_phys_page_range(start
, end
, 0);
1296 start
&= TARGET_PAGE_MASK
;
1297 start
+= TARGET_PAGE_SIZE
;
1302 * Invalidate all TBs which intersect with the target physical address range
1303 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1304 * 'is_cpu_write_access' should be true if called from a real cpu write
1305 * access: the virtual CPU will exit the current TB if code is modified inside
1308 * Called with mmap_lock held for user-mode emulation
1310 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1311 int is_cpu_write_access
)
1313 TranslationBlock
*tb
, *tb_next
;
1314 #if defined(TARGET_HAS_PRECISE_SMC)
1315 CPUState
*cpu
= current_cpu
;
1316 CPUArchState
*env
= NULL
;
1318 tb_page_addr_t tb_start
, tb_end
;
1321 #ifdef TARGET_HAS_PRECISE_SMC
1322 int current_tb_not_found
= is_cpu_write_access
;
1323 TranslationBlock
*current_tb
= NULL
;
1324 int current_tb_modified
= 0;
1325 target_ulong current_pc
= 0;
1326 target_ulong current_cs_base
= 0;
1327 uint32_t current_flags
= 0;
1328 #endif /* TARGET_HAS_PRECISE_SMC */
1330 p
= page_find(start
>> TARGET_PAGE_BITS
);
1334 #if defined(TARGET_HAS_PRECISE_SMC)
1340 /* we remove all the TBs in the range [start, end[ */
1341 /* XXX: see if in some cases it could be faster to invalidate all
1344 while (tb
!= NULL
) {
1345 n
= (uintptr_t)tb
& 3;
1346 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1347 tb_next
= tb
->page_next
[n
];
1348 /* NOTE: this is subtle as a TB may span two physical pages */
1350 /* NOTE: tb_end may be after the end of the page, but
1351 it is not a problem */
1352 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1353 tb_end
= tb_start
+ tb
->size
;
1355 tb_start
= tb
->page_addr
[1];
1356 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1358 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1359 #ifdef TARGET_HAS_PRECISE_SMC
1360 if (current_tb_not_found
) {
1361 current_tb_not_found
= 0;
1363 if (cpu
->mem_io_pc
) {
1364 /* now we have a real cpu fault */
1365 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1368 if (current_tb
== tb
&&
1369 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1370 /* If we are modifying the current TB, we must stop
1371 its execution. We could be more precise by checking
1372 that the modification is after the current PC, but it
1373 would require a specialized function to partially
1374 restore the CPU state */
1376 current_tb_modified
= 1;
1377 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1378 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1381 #endif /* TARGET_HAS_PRECISE_SMC */
1382 tb_phys_invalidate(tb
, -1);
1386 #if !defined(CONFIG_USER_ONLY)
1387 /* if no code remaining, no need to continue to use slow writes */
1389 invalidate_page_bitmap(p
);
1390 tlb_unprotect_code(start
);
1393 #ifdef TARGET_HAS_PRECISE_SMC
1394 if (current_tb_modified
) {
1395 /* we generate a block containing just the instruction
1396 modifying the memory. It will ensure that it cannot modify
1398 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1399 cpu_loop_exit_noexc(cpu
);
1404 #ifdef CONFIG_SOFTMMU
1405 /* len must be <= 8 and start must be a multiple of len */
1406 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1412 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1413 cpu_single_env
->mem_io_vaddr
, len
,
1414 cpu_single_env
->eip
,
1415 cpu_single_env
->eip
+
1416 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1419 p
= page_find(start
>> TARGET_PAGE_BITS
);
1423 if (!p
->code_bitmap
&&
1424 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1425 /* build code bitmap */
1426 build_page_bitmap(p
);
1428 if (p
->code_bitmap
) {
1432 nr
= start
& ~TARGET_PAGE_MASK
;
1433 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1434 if (b
& ((1 << len
) - 1)) {
1439 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1443 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1444 * host PC of the faulting store instruction that caused this invalidate.
1445 * Returns true if the caller needs to abort execution of the current
1446 * TB (because it was modified by this store and the guest CPU has
1447 * precise-SMC semantics).
1449 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
1451 TranslationBlock
*tb
;
1454 #ifdef TARGET_HAS_PRECISE_SMC
1455 TranslationBlock
*current_tb
= NULL
;
1456 CPUState
*cpu
= current_cpu
;
1457 CPUArchState
*env
= NULL
;
1458 int current_tb_modified
= 0;
1459 target_ulong current_pc
= 0;
1460 target_ulong current_cs_base
= 0;
1461 uint32_t current_flags
= 0;
1464 addr
&= TARGET_PAGE_MASK
;
1465 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1470 #ifdef TARGET_HAS_PRECISE_SMC
1471 if (tb
&& pc
!= 0) {
1472 current_tb
= tb_find_pc(pc
);
1478 while (tb
!= NULL
) {
1479 n
= (uintptr_t)tb
& 3;
1480 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1481 #ifdef TARGET_HAS_PRECISE_SMC
1482 if (current_tb
== tb
&&
1483 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1484 /* If we are modifying the current TB, we must stop
1485 its execution. We could be more precise by checking
1486 that the modification is after the current PC, but it
1487 would require a specialized function to partially
1488 restore the CPU state */
1490 current_tb_modified
= 1;
1491 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1492 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1495 #endif /* TARGET_HAS_PRECISE_SMC */
1496 tb_phys_invalidate(tb
, addr
);
1497 tb
= tb
->page_next
[n
];
1500 #ifdef TARGET_HAS_PRECISE_SMC
1501 if (current_tb_modified
) {
1502 /* we generate a block containing just the instruction
1503 modifying the memory. It will ensure that it cannot modify
1505 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1513 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1514 tb[1].tc_ptr. Return NULL if not found */
1515 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1517 int m_min
, m_max
, m
;
1519 TranslationBlock
*tb
;
1521 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1524 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1525 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1528 /* binary search (cf Knuth) */
1530 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1531 while (m_min
<= m_max
) {
1532 m
= (m_min
+ m_max
) >> 1;
1533 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1534 v
= (uintptr_t)tb
->tc_ptr
;
1537 } else if (tc_ptr
< v
) {
1543 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1546 #if !defined(CONFIG_USER_ONLY)
1547 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1549 ram_addr_t ram_addr
;
1554 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1555 if (!(memory_region_is_ram(mr
)
1556 || memory_region_is_romd(mr
))) {
1560 ram_addr
= memory_region_get_ram_addr(mr
) + addr
;
1561 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1564 #endif /* !defined(CONFIG_USER_ONLY) */
1566 void tb_check_watchpoint(CPUState
*cpu
)
1568 TranslationBlock
*tb
;
1570 tb
= tb_find_pc(cpu
->mem_io_pc
);
1572 /* We can use retranslation to find the PC. */
1573 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1574 tb_phys_invalidate(tb
, -1);
1576 /* The exception probably happened in a helper. The CPU state should
1577 have been saved before calling it. Fetch the PC from there. */
1578 CPUArchState
*env
= cpu
->env_ptr
;
1579 target_ulong pc
, cs_base
;
1580 tb_page_addr_t addr
;
1583 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1584 addr
= get_page_addr_code(env
, pc
);
1585 tb_invalidate_phys_range(addr
, addr
+ 1);
1589 #ifndef CONFIG_USER_ONLY
1590 /* in deterministic execution mode, instructions doing device I/Os
1591 must be at the end of the TB */
1592 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1594 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1595 CPUArchState
*env
= cpu
->env_ptr
;
1597 TranslationBlock
*tb
;
1599 target_ulong pc
, cs_base
;
1602 tb
= tb_find_pc(retaddr
);
1604 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1607 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1608 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1609 /* Calculate how many instructions had been executed before the fault
1611 n
= n
- cpu
->icount_decr
.u16
.low
;
1612 /* Generate a new TB ending on the I/O insn. */
1614 /* On MIPS and SH, delay slot instructions can only be restarted if
1615 they were already the first instruction in the TB. If this is not
1616 the first instruction in a TB then re-execute the preceding
1618 #if defined(TARGET_MIPS)
1619 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1620 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1621 cpu
->icount_decr
.u16
.low
++;
1622 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1624 #elif defined(TARGET_SH4)
1625 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1628 cpu
->icount_decr
.u16
.low
++;
1629 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1632 /* This should never happen. */
1633 if (n
> CF_COUNT_MASK
) {
1634 cpu_abort(cpu
, "TB too big during recompile");
1637 cflags
= n
| CF_LAST_IO
;
1639 cs_base
= tb
->cs_base
;
1641 tb_phys_invalidate(tb
, -1);
1642 if (tb
->cflags
& CF_NOCACHE
) {
1644 /* Invalidate original TB if this TB was generated in
1645 * cpu_exec_nocache() */
1646 tb_phys_invalidate(tb
->orig_tb
, -1);
1650 /* FIXME: In theory this could raise an exception. In practice
1651 we have already translated the block once so it's probably ok. */
1652 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1653 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1654 the first in the TB) then we end up generating a whole new TB and
1655 repeating the fault, which is horribly inefficient.
1656 Better would be to execute just this insn uncached, or generate a
1658 cpu_loop_exit_noexc(cpu
);
1661 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1665 /* Discard jump cache entries for any tb which might potentially
1666 overlap the flushed page. */
1667 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1668 memset(&cpu
->tb_jmp_cache
[i
], 0,
1669 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1671 i
= tb_jmp_cache_hash_page(addr
);
1672 memset(&cpu
->tb_jmp_cache
[i
], 0,
1673 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1676 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
1677 struct qht_stats hst
)
1679 uint32_t hgram_opts
;
1683 if (!hst
.head_buckets
) {
1686 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1687 hst
.used_head_buckets
, hst
.head_buckets
,
1688 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
1690 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1691 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
1692 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
1693 hgram_opts
|= QDIST_PR_NODECIMAL
;
1695 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
1696 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1697 qdist_avg(&hst
.occupancy
) * 100, hgram
);
1700 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1701 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
1702 if (hgram_bins
> 10) {
1706 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
1708 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
1709 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1710 qdist_avg(&hst
.chain
), hgram
);
1714 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1716 int i
, target_code_size
, max_target_code_size
;
1717 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1718 TranslationBlock
*tb
;
1719 struct qht_stats hst
;
1721 target_code_size
= 0;
1722 max_target_code_size
= 0;
1724 direct_jmp_count
= 0;
1725 direct_jmp2_count
= 0;
1726 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1727 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1728 target_code_size
+= tb
->size
;
1729 if (tb
->size
> max_target_code_size
) {
1730 max_target_code_size
= tb
->size
;
1732 if (tb
->page_addr
[1] != -1) {
1735 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1737 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1738 direct_jmp2_count
++;
1742 /* XXX: avoid using doubles ? */
1743 cpu_fprintf(f
, "Translation buffer state:\n");
1744 cpu_fprintf(f
, "gen code size %td/%zd\n",
1745 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1746 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1747 cpu_fprintf(f
, "TB count %d/%d\n",
1748 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1749 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1750 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1751 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1752 max_target_code_size
);
1753 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1754 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1755 tcg_ctx
.code_gen_buffer
) /
1756 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1757 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1758 tcg_ctx
.code_gen_buffer
) /
1759 target_code_size
: 0);
1760 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1761 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1762 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1763 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1765 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1766 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1768 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1769 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1771 qht_statistics_init(&tcg_ctx
.tb_ctx
.htable
, &hst
);
1772 print_qht_statistics(f
, cpu_fprintf
, hst
);
1773 qht_statistics_destroy(&hst
);
1775 cpu_fprintf(f
, "\nStatistics:\n");
1776 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1777 cpu_fprintf(f
, "TB invalidate count %d\n",
1778 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1779 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1780 tcg_dump_info(f
, cpu_fprintf
);
1783 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1785 tcg_dump_op_count(f
, cpu_fprintf
);
1788 #else /* CONFIG_USER_ONLY */
1790 void cpu_interrupt(CPUState
*cpu
, int mask
)
1792 cpu
->interrupt_request
|= mask
;
1793 cpu
->tcg_exit_req
= 1;
1797 * Walks guest process memory "regions" one by one
1798 * and calls callback function 'fn' for each region.
1800 struct walk_memory_regions_data
{
1801 walk_memory_regions_fn fn
;
1807 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1808 target_ulong end
, int new_prot
)
1810 if (data
->start
!= -1u) {
1811 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1817 data
->start
= (new_prot
? end
: -1u);
1818 data
->prot
= new_prot
;
1823 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1824 target_ulong base
, int level
, void **lp
)
1830 return walk_memory_regions_end(data
, base
, 0);
1836 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1837 int prot
= pd
[i
].flags
;
1839 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1840 if (prot
!= data
->prot
) {
1841 rc
= walk_memory_regions_end(data
, pa
, prot
);
1850 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1851 pa
= base
| ((target_ulong
)i
<<
1852 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1853 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1863 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1865 struct walk_memory_regions_data data
;
1873 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1874 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1875 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1881 return walk_memory_regions_end(&data
, 0, 0);
1884 static int dump_region(void *priv
, target_ulong start
,
1885 target_ulong end
, abi_ulong prot
)
1887 FILE *f
= (FILE *)priv
;
1889 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1890 " "TARGET_FMT_lx
" %c%c%c\n",
1891 start
, end
, end
- start
,
1892 ((prot
& PAGE_READ
) ? 'r' : '-'),
1893 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1894 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1899 /* dump memory mappings */
1900 void page_dump(FILE *f
)
1902 const int length
= sizeof(target_ulong
) * 2;
1903 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1904 length
, "start", length
, "end", length
, "size", "prot");
1905 walk_memory_regions(f
, dump_region
);
1908 int page_get_flags(target_ulong address
)
1912 p
= page_find(address
>> TARGET_PAGE_BITS
);
1919 /* Modify the flags of a page and invalidate the code if necessary.
1920 The flag PAGE_WRITE_ORG is positioned automatically depending
1921 on PAGE_WRITE. The mmap_lock should already be held. */
1922 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1924 target_ulong addr
, len
;
1926 /* This function should never be called with addresses outside the
1927 guest address space. If this assert fires, it probably indicates
1928 a missing call to h2g_valid. */
1929 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1930 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1932 assert(start
< end
);
1934 start
= start
& TARGET_PAGE_MASK
;
1935 end
= TARGET_PAGE_ALIGN(end
);
1937 if (flags
& PAGE_WRITE
) {
1938 flags
|= PAGE_WRITE_ORG
;
1941 for (addr
= start
, len
= end
- start
;
1943 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1944 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1946 /* If the write protection bit is set, then we invalidate
1948 if (!(p
->flags
& PAGE_WRITE
) &&
1949 (flags
& PAGE_WRITE
) &&
1951 tb_invalidate_phys_page(addr
, 0);
1957 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1963 /* This function should never be called with addresses outside the
1964 guest address space. If this assert fires, it probably indicates
1965 a missing call to h2g_valid. */
1966 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1967 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1973 if (start
+ len
- 1 < start
) {
1974 /* We've wrapped around. */
1978 /* must do before we loose bits in the next step */
1979 end
= TARGET_PAGE_ALIGN(start
+ len
);
1980 start
= start
& TARGET_PAGE_MASK
;
1982 for (addr
= start
, len
= end
- start
;
1984 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1985 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1989 if (!(p
->flags
& PAGE_VALID
)) {
1993 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1996 if (flags
& PAGE_WRITE
) {
1997 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2000 /* unprotect the page if it was put read-only because it
2001 contains translated code */
2002 if (!(p
->flags
& PAGE_WRITE
)) {
2003 if (!page_unprotect(addr
, 0)) {
2012 /* called from signal handler: invalidate the code and unprotect the
2013 * page. Return 0 if the fault was not handled, 1 if it was handled,
2014 * and 2 if it was handled but the caller must cause the TB to be
2015 * immediately exited. (We can only return 2 if the 'pc' argument is
2018 int page_unprotect(target_ulong address
, uintptr_t pc
)
2021 bool current_tb_invalidated
;
2023 target_ulong host_start
, host_end
, addr
;
2025 /* Technically this isn't safe inside a signal handler. However we
2026 know this only ever happens in a synchronous SEGV handler, so in
2027 practice it seems to be ok. */
2030 p
= page_find(address
>> TARGET_PAGE_BITS
);
2036 /* if the page was really writable, then we change its
2037 protection back to writable */
2038 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2039 host_start
= address
& qemu_host_page_mask
;
2040 host_end
= host_start
+ qemu_host_page_size
;
2043 current_tb_invalidated
= false;
2044 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2045 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2046 p
->flags
|= PAGE_WRITE
;
2049 /* and since the content will be modified, we must invalidate
2050 the corresponding translated code. */
2051 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2052 #ifdef DEBUG_TB_CHECK
2053 tb_invalidate_check(addr
);
2056 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2060 /* If current TB was invalidated return to main loop */
2061 return current_tb_invalidated
? 2 : 1;
2066 #endif /* CONFIG_USER_ONLY */