4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
24 #include "qemu/osdep.h"
27 #include "qemu-common.h"
28 #define NO_CPU_IO_DEFS
31 #include "disas/disas.h"
33 #if defined(CONFIG_USER_ONLY)
35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
36 #include <sys/param.h>
37 #if __FreeBSD_version >= 700104
38 #define HAVE_KINFO_GETVMMAP
39 #define sigqueue sigqueue_freebsd /* avoid redefinition */
41 #include <machine/profile.h>
50 #include "exec/address-spaces.h"
53 #include "exec/cputlb.h"
54 #include "exec/tb-hash.h"
55 #include "translate-all.h"
56 #include "qemu/bitmap.h"
57 #include "qemu/timer.h"
60 //#define DEBUG_TB_INVALIDATE
62 /* make various TB consistency checks */
63 //#define DEBUG_TB_CHECK
65 #if !defined(CONFIG_USER_ONLY)
66 /* TB consistency checks only implemented for usermode emulation. */
70 #define SMC_BITMAP_USE_THRESHOLD 10
72 typedef struct PageDesc
{
73 /* list of TBs intersecting this ram page */
74 TranslationBlock
*first_tb
;
76 /* in order to optimize self modifying code, we count the number
77 of lookups we do to a given page to use a bitmap */
78 unsigned int code_write_count
;
79 unsigned long *code_bitmap
;
85 /* In system mode we want L1_MAP to be based on ram offsets,
86 while in user mode we want it to be based on virtual addresses. */
87 #if !defined(CONFIG_USER_ONLY)
88 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
89 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
91 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
94 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
97 /* Size of the L2 (and L3, etc) page tables. */
99 #define V_L2_SIZE (1 << V_L2_BITS)
101 /* The bits remaining after N lower levels of page tables. */
102 #define V_L1_BITS_REM \
103 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
105 #if V_L1_BITS_REM < 4
106 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
108 #define V_L1_BITS V_L1_BITS_REM
111 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
113 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
115 uintptr_t qemu_host_page_size
;
116 intptr_t qemu_host_page_mask
;
118 /* The bottom level has pointers to PageDesc */
119 static void *l1_map
[V_L1_SIZE
];
121 /* code generation context */
124 /* translation block context */
125 #ifdef CONFIG_USER_ONLY
126 __thread
int have_tb_lock
;
131 #ifdef CONFIG_USER_ONLY
132 assert(!have_tb_lock
);
133 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
140 #ifdef CONFIG_USER_ONLY
141 assert(have_tb_lock
);
143 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
147 void tb_lock_reset(void)
149 #ifdef CONFIG_USER_ONLY
151 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
157 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
159 void cpu_gen_init(void)
161 tcg_context_init(&tcg_ctx
);
164 /* Encode VAL as a signed leb128 sequence at P.
165 Return P incremented past the encoded value. */
166 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
173 more
= !((val
== 0 && (byte
& 0x40) == 0)
174 || (val
== -1 && (byte
& 0x40) != 0));
184 /* Decode a signed leb128 sequence at *PP; increment *PP past the
185 decoded value. Return the decoded value. */
186 static target_long
decode_sleb128(uint8_t **pp
)
194 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
196 } while (byte
& 0x80);
197 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
198 val
|= -(target_ulong
)1 << shift
;
205 /* Encode the data collected about the instructions while compiling TB.
206 Place the data at BLOCK, and return the number of bytes consumed.
208 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
209 which come from the target's insn_start data, followed by a uintptr_t
210 which comes from the host pc of the end of the code implementing the insn.
212 Each line of the table is encoded as sleb128 deltas from the previous
213 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
214 That is, the first column is seeded with the guest pc, the last column
215 with the host pc, and the middle columns with zeros. */
217 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
219 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
223 tb
->tc_search
= block
;
225 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
228 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
230 prev
= (j
== 0 ? tb
->pc
: 0);
232 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
234 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
236 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
237 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
239 /* Test for (pending) buffer overflow. The assumption is that any
240 one row beginning below the high water mark cannot overrun
241 the buffer completely. Thus we can test for overflow after
242 encoding a row without having to check during encoding. */
243 if (unlikely(p
> highwater
)) {
251 /* The cpu state corresponding to 'searched_pc' is restored. */
252 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
253 uintptr_t searched_pc
)
255 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
256 uintptr_t host_pc
= (uintptr_t)tb
->tc_ptr
;
257 CPUArchState
*env
= cpu
->env_ptr
;
258 uint8_t *p
= tb
->tc_search
;
259 int i
, j
, num_insns
= tb
->icount
;
260 #ifdef CONFIG_PROFILER
261 int64_t ti
= profile_getclock();
264 if (searched_pc
< host_pc
) {
268 /* Reconstruct the stored insn data while looking for the point at
269 which the end of the insn exceeds the searched_pc. */
270 for (i
= 0; i
< num_insns
; ++i
) {
271 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
272 data
[j
] += decode_sleb128(&p
);
274 host_pc
+= decode_sleb128(&p
);
275 if (host_pc
> searched_pc
) {
282 if (tb
->cflags
& CF_USE_ICOUNT
) {
284 /* Reset the cycle counter to the start of the block. */
285 cpu
->icount_decr
.u16
.low
+= num_insns
;
286 /* Clear the IO flag. */
289 cpu
->icount_decr
.u16
.low
-= i
;
290 restore_state_to_opc(env
, tb
, data
);
292 #ifdef CONFIG_PROFILER
293 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
294 tcg_ctx
.restore_count
++;
299 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
301 TranslationBlock
*tb
;
303 tb
= tb_find_pc(retaddr
);
305 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
306 if (tb
->cflags
& CF_NOCACHE
) {
307 /* one-shot translation, invalidate it immediately */
308 cpu
->current_tb
= NULL
;
309 tb_phys_invalidate(tb
, -1);
317 void page_size_init(void)
319 /* NOTE: we can always suppose that qemu_host_page_size >=
321 qemu_real_host_page_size
= getpagesize();
322 qemu_real_host_page_mask
= -(intptr_t)qemu_real_host_page_size
;
323 if (qemu_host_page_size
== 0) {
324 qemu_host_page_size
= qemu_real_host_page_size
;
326 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
327 qemu_host_page_size
= TARGET_PAGE_SIZE
;
329 qemu_host_page_mask
= -(intptr_t)qemu_host_page_size
;
332 static void page_init(void)
335 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
337 #ifdef HAVE_KINFO_GETVMMAP
338 struct kinfo_vmentry
*freep
;
341 freep
= kinfo_getvmmap(getpid(), &cnt
);
344 for (i
= 0; i
< cnt
; i
++) {
345 unsigned long startaddr
, endaddr
;
347 startaddr
= freep
[i
].kve_start
;
348 endaddr
= freep
[i
].kve_end
;
349 if (h2g_valid(startaddr
)) {
350 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
352 if (h2g_valid(endaddr
)) {
353 endaddr
= h2g(endaddr
);
354 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
356 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
358 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
369 last_brk
= (unsigned long)sbrk(0);
371 f
= fopen("/compat/linux/proc/self/maps", "r");
376 unsigned long startaddr
, endaddr
;
379 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
381 if (n
== 2 && h2g_valid(startaddr
)) {
382 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
384 if (h2g_valid(endaddr
)) {
385 endaddr
= h2g(endaddr
);
389 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
402 * Called with mmap_lock held for user-mode emulation.
404 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
410 /* Level 1. Always allocated. */
411 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
414 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
415 void **p
= atomic_rcu_read(lp
);
421 p
= g_new0(void *, V_L2_SIZE
);
422 atomic_rcu_set(lp
, p
);
425 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
428 pd
= atomic_rcu_read(lp
);
433 pd
= g_new0(PageDesc
, V_L2_SIZE
);
434 atomic_rcu_set(lp
, pd
);
437 return pd
+ (index
& (V_L2_SIZE
- 1));
440 static inline PageDesc
*page_find(tb_page_addr_t index
)
442 return page_find_alloc(index
, 0);
445 #if defined(CONFIG_USER_ONLY)
446 /* Currently it is not recommended to allocate big chunks of data in
447 user mode. It will change when a dedicated libc will be used. */
448 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
449 region in which the guest needs to run. Revisit this. */
450 #define USE_STATIC_CODE_GEN_BUFFER
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__powerpc64__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
466 #elif defined(__powerpc__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
468 #elif defined(__aarch64__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
470 #elif defined(__arm__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
472 #elif defined(__s390x__)
473 /* We have a +- 4GB range on the branches; leave some slop. */
474 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
475 #elif defined(__mips__)
476 /* We have a 256MB branch region, but leave room to make sure the
477 main executable is also within that region. */
478 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
480 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
485 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
486 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
487 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
489 static inline size_t size_code_gen_buffer(size_t tb_size
)
491 /* Size the buffer. */
493 #ifdef USE_STATIC_CODE_GEN_BUFFER
494 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
496 /* ??? Needs adjustments. */
497 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
498 static buffer, we could size this on RESERVED_VA, on the text
499 segment size of the executable, or continue to use the default. */
500 tb_size
= (unsigned long)(ram_size
/ 4);
503 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
504 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
506 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
507 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
513 /* In order to use J and JAL within the code_gen_buffer, we require
514 that the buffer not cross a 256MB boundary. */
515 static inline bool cross_256mb(void *addr
, size_t size
)
517 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
520 /* We weren't able to allocate a buffer without crossing that boundary,
521 so make do with the larger portion of the buffer that doesn't cross.
522 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
523 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
525 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
526 size_t size2
= buf1
+ size1
- buf2
;
534 tcg_ctx
.code_gen_buffer_size
= size1
;
539 #ifdef USE_STATIC_CODE_GEN_BUFFER
540 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
541 __attribute__((aligned(CODE_GEN_ALIGN
)));
544 static inline void do_protect(void *addr
, long size
, int prot
)
547 VirtualProtect(addr
, size
, prot
, &old_protect
);
550 static inline void map_exec(void *addr
, long size
)
552 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
555 static inline void map_none(void *addr
, long size
)
557 do_protect(addr
, size
, PAGE_NOACCESS
);
560 static inline void do_protect(void *addr
, long size
, int prot
)
562 uintptr_t start
, end
;
564 start
= (uintptr_t)addr
;
565 start
&= qemu_real_host_page_mask
;
567 end
= (uintptr_t)addr
+ size
;
568 end
= ROUND_UP(end
, qemu_real_host_page_size
);
570 mprotect((void *)start
, end
- start
, prot
);
573 static inline void map_exec(void *addr
, long size
)
575 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
578 static inline void map_none(void *addr
, long size
)
580 do_protect(addr
, size
, PROT_NONE
);
584 static inline void *alloc_code_gen_buffer(void)
586 void *buf
= static_code_gen_buffer
;
587 size_t full_size
, size
;
589 /* The size of the buffer, rounded down to end on a page boundary. */
590 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
591 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
593 /* Reserve a guard page. */
594 size
= full_size
- qemu_real_host_page_size
;
596 /* Honor a command-line option limiting the size of the buffer. */
597 if (size
> tcg_ctx
.code_gen_buffer_size
) {
598 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
599 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
601 tcg_ctx
.code_gen_buffer_size
= size
;
604 if (cross_256mb(buf
, size
)) {
605 buf
= split_cross_256mb(buf
, size
);
606 size
= tcg_ctx
.code_gen_buffer_size
;
611 map_none(buf
+ size
, qemu_real_host_page_size
);
612 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
616 #elif defined(_WIN32)
617 static inline void *alloc_code_gen_buffer(void)
619 size_t size
= tcg_ctx
.code_gen_buffer_size
;
622 /* Perform the allocation in two steps, so that the guard page
623 is reserved but uncommitted. */
624 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
625 MEM_RESERVE
, PAGE_NOACCESS
);
627 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
628 assert(buf1
== buf2
);
634 static inline void *alloc_code_gen_buffer(void)
636 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
638 size_t size
= tcg_ctx
.code_gen_buffer_size
;
641 /* Constrain the position of the buffer based on the host cpu.
642 Note that these addresses are chosen in concert with the
643 addresses assigned in the relevant linker script file. */
644 # if defined(__PIE__) || defined(__PIC__)
645 /* Don't bother setting a preferred location if we're building
646 a position-independent executable. We're more likely to get
647 an address near the main executable if we let the kernel
648 choose the address. */
649 # elif defined(__x86_64__) && defined(MAP_32BIT)
650 /* Force the memory down into low memory with the executable.
651 Leave the choice of exact location with the kernel. */
653 /* Cannot expect to map more than 800MB in low memory. */
654 if (size
> 800u * 1024 * 1024) {
655 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
657 # elif defined(__sparc__)
658 start
= 0x40000000ul
;
659 # elif defined(__s390x__)
660 start
= 0x90000000ul
;
661 # elif defined(__mips__)
662 # if _MIPS_SIM == _ABI64
663 start
= 0x128000000ul
;
665 start
= 0x08000000ul
;
669 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
670 PROT_NONE
, flags
, -1, 0);
671 if (buf
== MAP_FAILED
) {
676 if (cross_256mb(buf
, size
)) {
677 /* Try again, with the original still mapped, to avoid re-acquiring
678 that 256mb crossing. This time don't specify an address. */
680 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
681 PROT_NONE
, flags
, -1, 0);
682 switch (buf2
!= MAP_FAILED
) {
684 if (!cross_256mb(buf2
, size
)) {
685 /* Success! Use the new buffer. */
686 munmap(buf
, size
+ qemu_real_host_page_size
);
689 /* Failure. Work with what we had. */
690 munmap(buf2
, size
+ qemu_real_host_page_size
);
693 /* Split the original buffer. Free the smaller half. */
694 buf2
= split_cross_256mb(buf
, size
);
695 size2
= tcg_ctx
.code_gen_buffer_size
;
697 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
699 munmap(buf
, size
- size2
);
708 /* Make the final buffer accessible. The guard page at the end
709 will remain inaccessible with PROT_NONE. */
710 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
712 /* Request large pages for the buffer. */
713 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
717 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
719 static inline void code_gen_alloc(size_t tb_size
)
721 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
722 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
723 if (tcg_ctx
.code_gen_buffer
== NULL
) {
724 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
728 /* Estimate a good size for the number of TBs we can support. We
729 still haven't deducted the prologue from the buffer size here,
730 but that's minimal and won't affect the estimate much. */
731 tcg_ctx
.code_gen_max_blocks
732 = tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
733 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
, tcg_ctx
.code_gen_max_blocks
);
735 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
738 /* Must be called before using the QEMU cpus. 'tb_size' is the size
739 (in bytes) allocated to the translation buffer. Zero means default
741 void tcg_exec_init(unsigned long tb_size
)
745 code_gen_alloc(tb_size
);
746 #if defined(CONFIG_SOFTMMU)
747 /* There's no guest base to take into account, so go ahead and
748 initialize the prologue now. */
749 tcg_prologue_init(&tcg_ctx
);
753 bool tcg_enabled(void)
755 return tcg_ctx
.code_gen_buffer
!= NULL
;
758 /* Allocate a new translation block. Flush the translation buffer if
759 too many translation blocks or too much generated code. */
760 static TranslationBlock
*tb_alloc(target_ulong pc
)
762 TranslationBlock
*tb
;
764 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
) {
767 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
773 void tb_free(TranslationBlock
*tb
)
775 /* In practice this is mostly used for single use temporary TB
776 Ignore the hard cases and just back up if this TB happens to
777 be the last one generated. */
778 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
779 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
780 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
781 tcg_ctx
.tb_ctx
.nb_tbs
--;
785 static inline void invalidate_page_bitmap(PageDesc
*p
)
787 #ifdef CONFIG_SOFTMMU
788 g_free(p
->code_bitmap
);
789 p
->code_bitmap
= NULL
;
790 p
->code_write_count
= 0;
794 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
795 static void page_flush_tb_1(int level
, void **lp
)
805 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
806 pd
[i
].first_tb
= NULL
;
807 invalidate_page_bitmap(pd
+ i
);
812 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
813 page_flush_tb_1(level
- 1, pp
+ i
);
818 static void page_flush_tb(void)
822 for (i
= 0; i
< V_L1_SIZE
; i
++) {
823 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
827 /* flush all the translation blocks */
828 /* XXX: tb_flush is currently not thread safe */
829 void tb_flush(CPUState
*cpu
)
831 #if defined(DEBUG_FLUSH)
832 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
833 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
834 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
835 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
836 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
838 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
839 > tcg_ctx
.code_gen_buffer_size
) {
840 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
842 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
845 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
846 cpu
->tb_flushed
= true;
849 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
852 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
853 /* XXX: flush processor icache at this point if cache flush is
855 tcg_ctx
.tb_ctx
.tb_flush_count
++;
858 #ifdef DEBUG_TB_CHECK
860 static void tb_invalidate_check(target_ulong address
)
862 TranslationBlock
*tb
;
865 address
&= TARGET_PAGE_MASK
;
866 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
867 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
868 tb
= tb
->phys_hash_next
) {
869 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
870 address
>= tb
->pc
+ tb
->size
)) {
871 printf("ERROR invalidate: address=" TARGET_FMT_lx
872 " PC=%08lx size=%04x\n",
873 address
, (long)tb
->pc
, tb
->size
);
879 /* verify that all the pages have correct rights for code */
880 static void tb_page_check(void)
882 TranslationBlock
*tb
;
883 int i
, flags1
, flags2
;
885 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
886 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
887 tb
= tb
->phys_hash_next
) {
888 flags1
= page_get_flags(tb
->pc
);
889 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
890 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
891 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
892 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
900 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
902 TranslationBlock
*tb1
;
907 *ptb
= tb1
->phys_hash_next
;
910 ptb
= &tb1
->phys_hash_next
;
914 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
916 TranslationBlock
*tb1
;
921 n1
= (uintptr_t)tb1
& 3;
922 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
924 *ptb
= tb1
->page_next
[n1
];
927 ptb
= &tb1
->page_next
[n1
];
931 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
932 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
934 TranslationBlock
*tb1
;
938 ptb
= &tb
->jmp_list_next
[n
];
940 /* find tb(n) in circular list */
944 tb1
= (TranslationBlock
*)(ntb
& ~3);
945 if (n1
== n
&& tb1
== tb
) {
949 ptb
= &tb1
->jmp_list_first
;
951 ptb
= &tb1
->jmp_list_next
[n1
];
954 /* now we can suppress tb(n) from the list */
955 *ptb
= tb
->jmp_list_next
[n
];
957 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
961 /* reset the jump entry 'n' of a TB so that it is not chained to
963 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
965 uintptr_t addr
= (uintptr_t)(tb
->tc_ptr
+ tb
->jmp_reset_offset
[n
]);
966 tb_set_jmp_target(tb
, n
, addr
);
969 /* remove any jumps to the TB */
970 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
972 TranslationBlock
*tb1
;
976 ptb
= &tb
->jmp_list_first
;
980 tb1
= (TranslationBlock
*)(ntb
& ~3);
984 tb_reset_jump(tb1
, n1
);
985 *ptb
= tb1
->jmp_list_next
[n1
];
986 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
990 /* invalidate one TB */
991 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
996 tb_page_addr_t phys_pc
;
998 /* remove the TB from the hash list */
999 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1000 h
= tb_phys_hash_func(phys_pc
);
1001 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
1003 /* remove the TB from the page list */
1004 if (tb
->page_addr
[0] != page_addr
) {
1005 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1006 tb_page_remove(&p
->first_tb
, tb
);
1007 invalidate_page_bitmap(p
);
1009 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1010 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1011 tb_page_remove(&p
->first_tb
, tb
);
1012 invalidate_page_bitmap(p
);
1015 /* remove the TB from the hash list */
1016 h
= tb_jmp_cache_hash_func(tb
->pc
);
1018 if (cpu
->tb_jmp_cache
[h
] == tb
) {
1019 cpu
->tb_jmp_cache
[h
] = NULL
;
1023 /* suppress this TB from the two jump lists */
1024 tb_remove_from_jmp_list(tb
, 0);
1025 tb_remove_from_jmp_list(tb
, 1);
1027 /* suppress any remaining jumps to this TB */
1030 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1033 #ifdef CONFIG_SOFTMMU
1034 static void build_page_bitmap(PageDesc
*p
)
1036 int n
, tb_start
, tb_end
;
1037 TranslationBlock
*tb
;
1039 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1042 while (tb
!= NULL
) {
1043 n
= (uintptr_t)tb
& 3;
1044 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1045 /* NOTE: this is subtle as a TB may span two physical pages */
1047 /* NOTE: tb_end may be after the end of the page, but
1048 it is not a problem */
1049 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1050 tb_end
= tb_start
+ tb
->size
;
1051 if (tb_end
> TARGET_PAGE_SIZE
) {
1052 tb_end
= TARGET_PAGE_SIZE
;
1056 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1058 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1059 tb
= tb
->page_next
[n
];
1064 /* add the tb in the target page and protect it if necessary
1066 * Called with mmap_lock held for user-mode emulation.
1068 static inline void tb_alloc_page(TranslationBlock
*tb
,
1069 unsigned int n
, tb_page_addr_t page_addr
)
1072 #ifndef CONFIG_USER_ONLY
1073 bool page_already_protected
;
1076 tb
->page_addr
[n
] = page_addr
;
1077 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1078 tb
->page_next
[n
] = p
->first_tb
;
1079 #ifndef CONFIG_USER_ONLY
1080 page_already_protected
= p
->first_tb
!= NULL
;
1082 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1083 invalidate_page_bitmap(p
);
1085 #if defined(CONFIG_USER_ONLY)
1086 if (p
->flags
& PAGE_WRITE
) {
1091 /* force the host page as non writable (writes will have a
1092 page fault + mprotect overhead) */
1093 page_addr
&= qemu_host_page_mask
;
1095 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1096 addr
+= TARGET_PAGE_SIZE
) {
1098 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1103 p2
->flags
&= ~PAGE_WRITE
;
1105 mprotect(g2h(page_addr
), qemu_host_page_size
,
1106 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1107 #ifdef DEBUG_TB_INVALIDATE
1108 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1113 /* if some code is already present, then the pages are already
1114 protected. So we handle the case where only the first TB is
1115 allocated in a physical page */
1116 if (!page_already_protected
) {
1117 tlb_protect_code(page_addr
);
1122 /* add a new TB and link it to the physical page tables. phys_page2 is
1123 * (-1) to indicate that only one page contains the TB.
1125 * Called with mmap_lock held for user-mode emulation.
1127 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1128 tb_page_addr_t phys_page2
)
1131 TranslationBlock
**ptb
;
1133 /* add in the physical hash table */
1134 h
= tb_phys_hash_func(phys_pc
);
1135 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1136 tb
->phys_hash_next
= *ptb
;
1139 /* add in the page list */
1140 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1141 if (phys_page2
!= -1) {
1142 tb_alloc_page(tb
, 1, phys_page2
);
1144 tb
->page_addr
[1] = -1;
1147 #ifdef DEBUG_TB_CHECK
1152 /* Called with mmap_lock held for user mode emulation. */
1153 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1154 target_ulong pc
, target_ulong cs_base
,
1155 uint32_t flags
, int cflags
)
1157 CPUArchState
*env
= cpu
->env_ptr
;
1158 TranslationBlock
*tb
;
1159 tb_page_addr_t phys_pc
, phys_page2
;
1160 target_ulong virt_page2
;
1161 tcg_insn_unit
*gen_code_buf
;
1162 int gen_code_size
, search_size
;
1163 #ifdef CONFIG_PROFILER
1167 phys_pc
= get_page_addr_code(env
, pc
);
1168 if (use_icount
&& !(cflags
& CF_IGNORE_ICOUNT
)) {
1169 cflags
|= CF_USE_ICOUNT
;
1173 if (unlikely(!tb
)) {
1175 /* flush must be done */
1177 /* cannot fail at this point */
1182 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1183 tb
->tc_ptr
= gen_code_buf
;
1184 tb
->cs_base
= cs_base
;
1186 tb
->cflags
= cflags
;
1188 #ifdef CONFIG_PROFILER
1189 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1191 ti
= profile_getclock();
1194 tcg_func_start(&tcg_ctx
);
1196 gen_intermediate_code(env
, tb
);
1198 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
1200 /* generate machine code */
1201 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1202 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1203 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1204 #ifdef USE_DIRECT_JUMP
1205 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_insn_offset
;
1206 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1208 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1209 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_addr
;
1212 #ifdef CONFIG_PROFILER
1214 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1215 tcg_ctx
.code_time
-= profile_getclock();
1218 /* ??? Overflow could be handled better here. In particular, we
1219 don't need to re-do gen_intermediate_code, nor should we re-do
1220 the tcg optimization currently hidden inside tcg_gen_code. All
1221 that should be required is to flush the TBs, allocate a new TB,
1222 re-initialize it per above, and re-do the actual code generation. */
1223 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1224 if (unlikely(gen_code_size
< 0)) {
1225 goto buffer_overflow
;
1227 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1228 if (unlikely(search_size
< 0)) {
1229 goto buffer_overflow
;
1232 #ifdef CONFIG_PROFILER
1233 tcg_ctx
.code_time
+= profile_getclock();
1234 tcg_ctx
.code_in_len
+= tb
->size
;
1235 tcg_ctx
.code_out_len
+= gen_code_size
;
1236 tcg_ctx
.search_out_len
+= search_size
;
1240 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1241 qemu_log_in_addr_range(tb
->pc
)) {
1242 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1243 log_disas(tb
->tc_ptr
, gen_code_size
);
1249 tcg_ctx
.code_gen_ptr
= (void *)
1250 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1253 /* init jump list */
1254 assert(((uintptr_t)tb
& 3) == 0);
1255 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1256 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1257 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1259 /* init original jump addresses wich has been set during tcg_gen_code() */
1260 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1261 tb_reset_jump(tb
, 0);
1263 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1264 tb_reset_jump(tb
, 1);
1267 /* check next page if needed */
1268 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1270 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1271 phys_page2
= get_page_addr_code(env
, virt_page2
);
1273 /* As long as consistency of the TB stuff is provided by tb_lock in user
1274 * mode and is implicit in single-threaded softmmu emulation, no explicit
1275 * memory barrier is required before tb_link_page() makes the TB visible
1276 * through the physical hash table and physical page list.
1278 tb_link_page(tb
, phys_pc
, phys_page2
);
1283 * Invalidate all TBs which intersect with the target physical address range
1284 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1285 * 'is_cpu_write_access' should be true if called from a real cpu write
1286 * access: the virtual CPU will exit the current TB if code is modified inside
1289 * Called with mmap_lock held for user-mode emulation
1291 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1293 while (start
< end
) {
1294 tb_invalidate_phys_page_range(start
, end
, 0);
1295 start
&= TARGET_PAGE_MASK
;
1296 start
+= TARGET_PAGE_SIZE
;
1301 * Invalidate all TBs which intersect with the target physical address range
1302 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1303 * 'is_cpu_write_access' should be true if called from a real cpu write
1304 * access: the virtual CPU will exit the current TB if code is modified inside
1307 * Called with mmap_lock held for user-mode emulation
1309 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1310 int is_cpu_write_access
)
1312 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1313 CPUState
*cpu
= current_cpu
;
1314 #if defined(TARGET_HAS_PRECISE_SMC)
1315 CPUArchState
*env
= NULL
;
1317 tb_page_addr_t tb_start
, tb_end
;
1320 #ifdef TARGET_HAS_PRECISE_SMC
1321 int current_tb_not_found
= is_cpu_write_access
;
1322 TranslationBlock
*current_tb
= NULL
;
1323 int current_tb_modified
= 0;
1324 target_ulong current_pc
= 0;
1325 target_ulong current_cs_base
= 0;
1326 uint32_t current_flags
= 0;
1327 #endif /* TARGET_HAS_PRECISE_SMC */
1329 p
= page_find(start
>> TARGET_PAGE_BITS
);
1333 #if defined(TARGET_HAS_PRECISE_SMC)
1339 /* we remove all the TBs in the range [start, end[ */
1340 /* XXX: see if in some cases it could be faster to invalidate all
1343 while (tb
!= NULL
) {
1344 n
= (uintptr_t)tb
& 3;
1345 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1346 tb_next
= tb
->page_next
[n
];
1347 /* NOTE: this is subtle as a TB may span two physical pages */
1349 /* NOTE: tb_end may be after the end of the page, but
1350 it is not a problem */
1351 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1352 tb_end
= tb_start
+ tb
->size
;
1354 tb_start
= tb
->page_addr
[1];
1355 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1357 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1358 #ifdef TARGET_HAS_PRECISE_SMC
1359 if (current_tb_not_found
) {
1360 current_tb_not_found
= 0;
1362 if (cpu
->mem_io_pc
) {
1363 /* now we have a real cpu fault */
1364 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1367 if (current_tb
== tb
&&
1368 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1369 /* If we are modifying the current TB, we must stop
1370 its execution. We could be more precise by checking
1371 that the modification is after the current PC, but it
1372 would require a specialized function to partially
1373 restore the CPU state */
1375 current_tb_modified
= 1;
1376 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1377 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1380 #endif /* TARGET_HAS_PRECISE_SMC */
1381 /* we need to do that to handle the case where a signal
1382 occurs while doing tb_phys_invalidate() */
1385 saved_tb
= cpu
->current_tb
;
1386 cpu
->current_tb
= NULL
;
1388 tb_phys_invalidate(tb
, -1);
1390 cpu
->current_tb
= saved_tb
;
1391 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1392 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1398 #if !defined(CONFIG_USER_ONLY)
1399 /* if no code remaining, no need to continue to use slow writes */
1401 invalidate_page_bitmap(p
);
1402 tlb_unprotect_code(start
);
1405 #ifdef TARGET_HAS_PRECISE_SMC
1406 if (current_tb_modified
) {
1407 /* we generate a block containing just the instruction
1408 modifying the memory. It will ensure that it cannot modify
1410 cpu
->current_tb
= NULL
;
1411 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1412 cpu_resume_from_signal(cpu
, NULL
);
1417 #ifdef CONFIG_SOFTMMU
1418 /* len must be <= 8 and start must be a multiple of len */
1419 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1425 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1426 cpu_single_env
->mem_io_vaddr
, len
,
1427 cpu_single_env
->eip
,
1428 cpu_single_env
->eip
+
1429 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1432 p
= page_find(start
>> TARGET_PAGE_BITS
);
1436 if (!p
->code_bitmap
&&
1437 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1438 /* build code bitmap */
1439 build_page_bitmap(p
);
1441 if (p
->code_bitmap
) {
1445 nr
= start
& ~TARGET_PAGE_MASK
;
1446 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1447 if (b
& ((1 << len
) - 1)) {
1452 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1456 /* Called with mmap_lock held. */
1457 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1458 uintptr_t pc
, void *puc
,
1461 TranslationBlock
*tb
;
1464 #ifdef TARGET_HAS_PRECISE_SMC
1465 TranslationBlock
*current_tb
= NULL
;
1466 CPUState
*cpu
= current_cpu
;
1467 CPUArchState
*env
= NULL
;
1468 int current_tb_modified
= 0;
1469 target_ulong current_pc
= 0;
1470 target_ulong current_cs_base
= 0;
1471 uint32_t current_flags
= 0;
1474 addr
&= TARGET_PAGE_MASK
;
1475 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1480 #ifdef TARGET_HAS_PRECISE_SMC
1481 if (tb
&& pc
!= 0) {
1482 current_tb
= tb_find_pc(pc
);
1488 while (tb
!= NULL
) {
1489 n
= (uintptr_t)tb
& 3;
1490 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1491 #ifdef TARGET_HAS_PRECISE_SMC
1492 if (current_tb
== tb
&&
1493 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1494 /* If we are modifying the current TB, we must stop
1495 its execution. We could be more precise by checking
1496 that the modification is after the current PC, but it
1497 would require a specialized function to partially
1498 restore the CPU state */
1500 current_tb_modified
= 1;
1501 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1502 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1505 #endif /* TARGET_HAS_PRECISE_SMC */
1506 tb_phys_invalidate(tb
, addr
);
1507 tb
= tb
->page_next
[n
];
1510 #ifdef TARGET_HAS_PRECISE_SMC
1511 if (current_tb_modified
) {
1512 /* we generate a block containing just the instruction
1513 modifying the memory. It will ensure that it cannot modify
1515 cpu
->current_tb
= NULL
;
1516 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1520 cpu_resume_from_signal(cpu
, puc
);
1526 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1527 tb[1].tc_ptr. Return NULL if not found */
1528 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1530 int m_min
, m_max
, m
;
1532 TranslationBlock
*tb
;
1534 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1537 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1538 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1541 /* binary search (cf Knuth) */
1543 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1544 while (m_min
<= m_max
) {
1545 m
= (m_min
+ m_max
) >> 1;
1546 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1547 v
= (uintptr_t)tb
->tc_ptr
;
1550 } else if (tc_ptr
< v
) {
1556 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1559 #if !defined(CONFIG_USER_ONLY)
1560 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1562 ram_addr_t ram_addr
;
1567 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1568 if (!(memory_region_is_ram(mr
)
1569 || memory_region_is_romd(mr
))) {
1573 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1575 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1578 #endif /* !defined(CONFIG_USER_ONLY) */
1580 void tb_check_watchpoint(CPUState
*cpu
)
1582 TranslationBlock
*tb
;
1584 tb
= tb_find_pc(cpu
->mem_io_pc
);
1586 /* We can use retranslation to find the PC. */
1587 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1588 tb_phys_invalidate(tb
, -1);
1590 /* The exception probably happened in a helper. The CPU state should
1591 have been saved before calling it. Fetch the PC from there. */
1592 CPUArchState
*env
= cpu
->env_ptr
;
1593 target_ulong pc
, cs_base
;
1594 tb_page_addr_t addr
;
1597 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1598 addr
= get_page_addr_code(env
, pc
);
1599 tb_invalidate_phys_range(addr
, addr
+ 1);
1603 #ifndef CONFIG_USER_ONLY
1604 /* in deterministic execution mode, instructions doing device I/Os
1605 must be at the end of the TB */
1606 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1608 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1609 CPUArchState
*env
= cpu
->env_ptr
;
1611 TranslationBlock
*tb
;
1613 target_ulong pc
, cs_base
;
1616 tb
= tb_find_pc(retaddr
);
1618 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1621 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1622 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1623 /* Calculate how many instructions had been executed before the fault
1625 n
= n
- cpu
->icount_decr
.u16
.low
;
1626 /* Generate a new TB ending on the I/O insn. */
1628 /* On MIPS and SH, delay slot instructions can only be restarted if
1629 they were already the first instruction in the TB. If this is not
1630 the first instruction in a TB then re-execute the preceding
1632 #if defined(TARGET_MIPS)
1633 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1634 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1635 cpu
->icount_decr
.u16
.low
++;
1636 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1638 #elif defined(TARGET_SH4)
1639 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1642 cpu
->icount_decr
.u16
.low
++;
1643 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1646 /* This should never happen. */
1647 if (n
> CF_COUNT_MASK
) {
1648 cpu_abort(cpu
, "TB too big during recompile");
1651 cflags
= n
| CF_LAST_IO
;
1653 cs_base
= tb
->cs_base
;
1655 tb_phys_invalidate(tb
, -1);
1656 if (tb
->cflags
& CF_NOCACHE
) {
1658 /* Invalidate original TB if this TB was generated in
1659 * cpu_exec_nocache() */
1660 tb_phys_invalidate(tb
->orig_tb
, -1);
1664 /* FIXME: In theory this could raise an exception. In practice
1665 we have already translated the block once so it's probably ok. */
1666 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1667 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1668 the first in the TB) then we end up generating a whole new TB and
1669 repeating the fault, which is horribly inefficient.
1670 Better would be to execute just this insn uncached, or generate a
1672 cpu_resume_from_signal(cpu
, NULL
);
1675 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1679 /* Discard jump cache entries for any tb which might potentially
1680 overlap the flushed page. */
1681 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1682 memset(&cpu
->tb_jmp_cache
[i
], 0,
1683 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1685 i
= tb_jmp_cache_hash_page(addr
);
1686 memset(&cpu
->tb_jmp_cache
[i
], 0,
1687 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1690 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1692 int i
, target_code_size
, max_target_code_size
;
1693 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1694 TranslationBlock
*tb
;
1696 target_code_size
= 0;
1697 max_target_code_size
= 0;
1699 direct_jmp_count
= 0;
1700 direct_jmp2_count
= 0;
1701 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1702 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1703 target_code_size
+= tb
->size
;
1704 if (tb
->size
> max_target_code_size
) {
1705 max_target_code_size
= tb
->size
;
1707 if (tb
->page_addr
[1] != -1) {
1710 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1712 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1713 direct_jmp2_count
++;
1717 /* XXX: avoid using doubles ? */
1718 cpu_fprintf(f
, "Translation buffer state:\n");
1719 cpu_fprintf(f
, "gen code size %td/%zd\n",
1720 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1721 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1722 cpu_fprintf(f
, "TB count %d/%d\n",
1723 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1724 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1725 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1726 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1727 max_target_code_size
);
1728 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1729 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1730 tcg_ctx
.code_gen_buffer
) /
1731 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1732 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1733 tcg_ctx
.code_gen_buffer
) /
1734 target_code_size
: 0);
1735 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1736 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1737 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1738 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1740 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1741 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1743 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1744 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1745 cpu_fprintf(f
, "\nStatistics:\n");
1746 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1747 cpu_fprintf(f
, "TB invalidate count %d\n",
1748 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1749 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1750 tcg_dump_info(f
, cpu_fprintf
);
1753 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1755 tcg_dump_op_count(f
, cpu_fprintf
);
1758 #else /* CONFIG_USER_ONLY */
1760 void cpu_interrupt(CPUState
*cpu
, int mask
)
1762 cpu
->interrupt_request
|= mask
;
1763 cpu
->tcg_exit_req
= 1;
1767 * Walks guest process memory "regions" one by one
1768 * and calls callback function 'fn' for each region.
1770 struct walk_memory_regions_data
{
1771 walk_memory_regions_fn fn
;
1777 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1778 target_ulong end
, int new_prot
)
1780 if (data
->start
!= -1u) {
1781 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1787 data
->start
= (new_prot
? end
: -1u);
1788 data
->prot
= new_prot
;
1793 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1794 target_ulong base
, int level
, void **lp
)
1800 return walk_memory_regions_end(data
, base
, 0);
1806 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1807 int prot
= pd
[i
].flags
;
1809 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1810 if (prot
!= data
->prot
) {
1811 rc
= walk_memory_regions_end(data
, pa
, prot
);
1820 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1821 pa
= base
| ((target_ulong
)i
<<
1822 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1823 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1833 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1835 struct walk_memory_regions_data data
;
1843 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1844 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1845 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1851 return walk_memory_regions_end(&data
, 0, 0);
1854 static int dump_region(void *priv
, target_ulong start
,
1855 target_ulong end
, unsigned long prot
)
1857 FILE *f
= (FILE *)priv
;
1859 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1860 " "TARGET_FMT_lx
" %c%c%c\n",
1861 start
, end
, end
- start
,
1862 ((prot
& PAGE_READ
) ? 'r' : '-'),
1863 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1864 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1869 /* dump memory mappings */
1870 void page_dump(FILE *f
)
1872 const int length
= sizeof(target_ulong
) * 2;
1873 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1874 length
, "start", length
, "end", length
, "size", "prot");
1875 walk_memory_regions(f
, dump_region
);
1878 int page_get_flags(target_ulong address
)
1882 p
= page_find(address
>> TARGET_PAGE_BITS
);
1889 /* Modify the flags of a page and invalidate the code if necessary.
1890 The flag PAGE_WRITE_ORG is positioned automatically depending
1891 on PAGE_WRITE. The mmap_lock should already be held. */
1892 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1894 target_ulong addr
, len
;
1896 /* This function should never be called with addresses outside the
1897 guest address space. If this assert fires, it probably indicates
1898 a missing call to h2g_valid. */
1899 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1900 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1902 assert(start
< end
);
1904 start
= start
& TARGET_PAGE_MASK
;
1905 end
= TARGET_PAGE_ALIGN(end
);
1907 if (flags
& PAGE_WRITE
) {
1908 flags
|= PAGE_WRITE_ORG
;
1911 for (addr
= start
, len
= end
- start
;
1913 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1914 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1916 /* If the write protection bit is set, then we invalidate
1918 if (!(p
->flags
& PAGE_WRITE
) &&
1919 (flags
& PAGE_WRITE
) &&
1921 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1927 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1933 /* This function should never be called with addresses outside the
1934 guest address space. If this assert fires, it probably indicates
1935 a missing call to h2g_valid. */
1936 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1937 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1943 if (start
+ len
- 1 < start
) {
1944 /* We've wrapped around. */
1948 /* must do before we loose bits in the next step */
1949 end
= TARGET_PAGE_ALIGN(start
+ len
);
1950 start
= start
& TARGET_PAGE_MASK
;
1952 for (addr
= start
, len
= end
- start
;
1954 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1955 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1959 if (!(p
->flags
& PAGE_VALID
)) {
1963 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1966 if (flags
& PAGE_WRITE
) {
1967 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1970 /* unprotect the page if it was put read-only because it
1971 contains translated code */
1972 if (!(p
->flags
& PAGE_WRITE
)) {
1973 if (!page_unprotect(addr
, 0, NULL
)) {
1982 /* called from signal handler: invalidate the code and unprotect the
1983 page. Return TRUE if the fault was successfully handled. */
1984 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1988 target_ulong host_start
, host_end
, addr
;
1990 /* Technically this isn't safe inside a signal handler. However we
1991 know this only ever happens in a synchronous SEGV handler, so in
1992 practice it seems to be ok. */
1995 p
= page_find(address
>> TARGET_PAGE_BITS
);
2001 /* if the page was really writable, then we change its
2002 protection back to writable */
2003 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2004 host_start
= address
& qemu_host_page_mask
;
2005 host_end
= host_start
+ qemu_host_page_size
;
2008 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2009 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2010 p
->flags
|= PAGE_WRITE
;
2013 /* and since the content will be modified, we must invalidate
2014 the corresponding translated code. */
2015 tb_invalidate_phys_page(addr
, pc
, puc
, true);
2016 #ifdef DEBUG_TB_CHECK
2017 tb_invalidate_check(addr
);
2020 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2029 #endif /* CONFIG_USER_ONLY */