4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "qemu/osdep.h"
25 #include "qemu-common.h"
26 #define NO_CPU_IO_DEFS
29 #include "disas/disas.h"
30 #include "exec/exec-all.h"
32 #if defined(CONFIG_USER_ONLY)
34 #include "exec/exec-all.h"
35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
36 #include <sys/param.h>
37 #if __FreeBSD_version >= 700104
38 #define HAVE_KINFO_GETVMMAP
39 #define sigqueue sigqueue_freebsd /* avoid redefinition */
41 #include <machine/profile.h>
50 #include "exec/address-spaces.h"
53 #include "exec/cputlb.h"
54 #include "exec/tb-hash.h"
55 #include "translate-all.h"
56 #include "qemu/bitmap.h"
57 #include "qemu/error-report.h"
58 #include "qemu/timer.h"
59 #include "qemu/main-loop.h"
61 #include "sysemu/cpus.h"
63 /* #define DEBUG_TB_INVALIDATE */
64 /* #define DEBUG_TB_FLUSH */
65 /* make various TB consistency checks */
66 /* #define DEBUG_TB_CHECK */
69 #define DEBUG_TB_FLUSH_GATE 1
71 #define DEBUG_TB_FLUSH_GATE 0
74 #if !defined(CONFIG_USER_ONLY)
75 /* TB consistency checks only implemented for usermode emulation. */
79 /* Access to the various translations structures need to be serialised via locks
80 * for consistency. This is automatic for SoftMMU based system
81 * emulation due to its single threaded nature. In user-mode emulation
82 * access to the memory related structures are protected with the
86 #define assert_memory_lock() tcg_debug_assert(have_tb_lock)
88 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
91 #define SMC_BITMAP_USE_THRESHOLD 10
93 typedef struct PageDesc
{
94 /* list of TBs intersecting this ram page */
95 TranslationBlock
*first_tb
;
97 /* in order to optimize self modifying code, we count the number
98 of lookups we do to a given page to use a bitmap */
99 unsigned int code_write_count
;
100 unsigned long *code_bitmap
;
106 /* In system mode we want L1_MAP to be based on ram offsets,
107 while in user mode we want it to be based on virtual addresses. */
108 #if !defined(CONFIG_USER_ONLY)
109 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
110 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
112 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
115 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
118 /* Size of the L2 (and L3, etc) page tables. */
120 #define V_L2_SIZE (1 << V_L2_BITS)
122 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
123 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS
>
124 sizeof(((TranslationBlock
*)0)->trace_vcpu_dstate
)
128 * L1 Mapping properties
130 static int v_l1_size
;
131 static int v_l1_shift
;
132 static int v_l2_levels
;
134 /* The bottom level has pointers to PageDesc, and is indexed by
135 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
137 #define V_L1_MIN_BITS 4
138 #define V_L1_MAX_BITS (V_L2_BITS + 3)
139 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
141 static void *l1_map
[V_L1_MAX_SIZE
];
143 /* code generation context */
147 /* translation block context */
148 static __thread
int have_tb_lock
;
150 static void page_table_config_init(void)
154 assert(TARGET_PAGE_BITS
);
155 /* The bits remaining after N lower levels of page tables. */
156 v_l1_bits
= (L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
) % V_L2_BITS
;
157 if (v_l1_bits
< V_L1_MIN_BITS
) {
158 v_l1_bits
+= V_L2_BITS
;
161 v_l1_size
= 1 << v_l1_bits
;
162 v_l1_shift
= L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
- v_l1_bits
;
163 v_l2_levels
= v_l1_shift
/ V_L2_BITS
- 1;
165 assert(v_l1_bits
<= V_L1_MAX_BITS
);
166 assert(v_l1_shift
% V_L2_BITS
== 0);
167 assert(v_l2_levels
>= 0);
170 #define assert_tb_locked() tcg_debug_assert(have_tb_lock)
171 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock)
175 assert_tb_unlocked();
176 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
184 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
187 void tb_lock_reset(void)
190 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
195 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
197 void cpu_gen_init(void)
199 tcg_context_init(&tcg_ctx
);
202 /* Encode VAL as a signed leb128 sequence at P.
203 Return P incremented past the encoded value. */
204 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
211 more
= !((val
== 0 && (byte
& 0x40) == 0)
212 || (val
== -1 && (byte
& 0x40) != 0));
222 /* Decode a signed leb128 sequence at *PP; increment *PP past the
223 decoded value. Return the decoded value. */
224 static target_long
decode_sleb128(uint8_t **pp
)
232 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
234 } while (byte
& 0x80);
235 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
236 val
|= -(target_ulong
)1 << shift
;
243 /* Encode the data collected about the instructions while compiling TB.
244 Place the data at BLOCK, and return the number of bytes consumed.
246 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
247 which come from the target's insn_start data, followed by a uintptr_t
248 which comes from the host pc of the end of the code implementing the insn.
250 Each line of the table is encoded as sleb128 deltas from the previous
251 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
252 That is, the first column is seeded with the guest pc, the last column
253 with the host pc, and the middle columns with zeros. */
255 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
257 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
261 tb
->tc_search
= block
;
263 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
266 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
268 prev
= (j
== 0 ? tb
->pc
: 0);
270 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
272 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
274 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
275 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
277 /* Test for (pending) buffer overflow. The assumption is that any
278 one row beginning below the high water mark cannot overrun
279 the buffer completely. Thus we can test for overflow after
280 encoding a row without having to check during encoding. */
281 if (unlikely(p
> highwater
)) {
289 /* The cpu state corresponding to 'searched_pc' is restored.
290 * Called with tb_lock held.
292 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
293 uintptr_t searched_pc
)
295 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
296 uintptr_t host_pc
= (uintptr_t)tb
->tc_ptr
;
297 CPUArchState
*env
= cpu
->env_ptr
;
298 uint8_t *p
= tb
->tc_search
;
299 int i
, j
, num_insns
= tb
->icount
;
300 #ifdef CONFIG_PROFILER
301 int64_t ti
= profile_getclock();
304 searched_pc
-= GETPC_ADJ
;
306 if (searched_pc
< host_pc
) {
310 /* Reconstruct the stored insn data while looking for the point at
311 which the end of the insn exceeds the searched_pc. */
312 for (i
= 0; i
< num_insns
; ++i
) {
313 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
314 data
[j
] += decode_sleb128(&p
);
316 host_pc
+= decode_sleb128(&p
);
317 if (host_pc
> searched_pc
) {
324 if (tb
->cflags
& CF_USE_ICOUNT
) {
326 /* Reset the cycle counter to the start of the block. */
327 cpu
->icount_decr
.u16
.low
+= num_insns
;
328 /* Clear the IO flag. */
331 cpu
->icount_decr
.u16
.low
-= i
;
332 restore_state_to_opc(env
, tb
, data
);
334 #ifdef CONFIG_PROFILER
335 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
336 tcg_ctx
.restore_count
++;
341 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
343 TranslationBlock
*tb
;
346 /* A retaddr of zero is invalid so we really shouldn't have ended
347 * up here. The target code has likely forgotten to check retaddr
348 * != 0 before attempting to restore state. We return early to
349 * avoid blowing up on a recursive tb_lock(). The target must have
350 * previously survived a failed cpu_restore_state because
351 * tb_find_pc(0) would have failed anyway. It still should be
360 tb
= tb_find_pc(retaddr
);
362 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
363 if (tb
->cflags
& CF_NOCACHE
) {
364 /* one-shot translation, invalidate it immediately */
365 tb_phys_invalidate(tb
, -1);
375 static void page_init(void)
378 page_table_config_init();
380 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
382 #ifdef HAVE_KINFO_GETVMMAP
383 struct kinfo_vmentry
*freep
;
386 freep
= kinfo_getvmmap(getpid(), &cnt
);
389 for (i
= 0; i
< cnt
; i
++) {
390 unsigned long startaddr
, endaddr
;
392 startaddr
= freep
[i
].kve_start
;
393 endaddr
= freep
[i
].kve_end
;
394 if (h2g_valid(startaddr
)) {
395 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
397 if (h2g_valid(endaddr
)) {
398 endaddr
= h2g(endaddr
);
399 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
401 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
403 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
414 last_brk
= (unsigned long)sbrk(0);
416 f
= fopen("/compat/linux/proc/self/maps", "r");
421 unsigned long startaddr
, endaddr
;
424 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
426 if (n
== 2 && h2g_valid(startaddr
)) {
427 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
429 if (h2g_valid(endaddr
)) {
430 endaddr
= h2g(endaddr
);
434 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
447 * Called with tb_lock held for system emulation.
448 * Called with mmap_lock held for user-mode emulation.
450 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
457 assert_memory_lock();
460 /* Level 1. Always allocated. */
461 lp
= l1_map
+ ((index
>> v_l1_shift
) & (v_l1_size
- 1));
464 for (i
= v_l2_levels
; i
> 0; i
--) {
465 void **p
= atomic_rcu_read(lp
);
471 p
= g_new0(void *, V_L2_SIZE
);
472 atomic_rcu_set(lp
, p
);
475 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
478 pd
= atomic_rcu_read(lp
);
483 pd
= g_new0(PageDesc
, V_L2_SIZE
);
484 atomic_rcu_set(lp
, pd
);
487 return pd
+ (index
& (V_L2_SIZE
- 1));
490 static inline PageDesc
*page_find(tb_page_addr_t index
)
492 return page_find_alloc(index
, 0);
495 #if defined(CONFIG_USER_ONLY)
496 /* Currently it is not recommended to allocate big chunks of data in
497 user mode. It will change when a dedicated libc will be used. */
498 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
499 region in which the guest needs to run. Revisit this. */
500 #define USE_STATIC_CODE_GEN_BUFFER
503 /* Minimum size of the code gen buffer. This number is randomly chosen,
504 but not so small that we can't have a fair number of TB's live. */
505 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
507 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
508 indicated, this is constrained by the range of direct branches on the
509 host cpu, as used by the TCG implementation of goto_tb. */
510 #if defined(__x86_64__)
511 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
512 #elif defined(__sparc__)
513 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
514 #elif defined(__powerpc64__)
515 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
516 #elif defined(__powerpc__)
517 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
518 #elif defined(__aarch64__)
519 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
520 #elif defined(__s390x__)
521 /* We have a +- 4GB range on the branches; leave some slop. */
522 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
523 #elif defined(__mips__)
524 /* We have a 256MB branch region, but leave room to make sure the
525 main executable is also within that region. */
526 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
528 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
531 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
533 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
534 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
535 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
537 static inline size_t size_code_gen_buffer(size_t tb_size
)
539 /* Size the buffer. */
541 #ifdef USE_STATIC_CODE_GEN_BUFFER
542 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
544 /* ??? Needs adjustments. */
545 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
546 static buffer, we could size this on RESERVED_VA, on the text
547 segment size of the executable, or continue to use the default. */
548 tb_size
= (unsigned long)(ram_size
/ 4);
551 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
552 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
554 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
555 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
561 /* In order to use J and JAL within the code_gen_buffer, we require
562 that the buffer not cross a 256MB boundary. */
563 static inline bool cross_256mb(void *addr
, size_t size
)
565 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
568 /* We weren't able to allocate a buffer without crossing that boundary,
569 so make do with the larger portion of the buffer that doesn't cross.
570 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
571 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
573 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
574 size_t size2
= buf1
+ size1
- buf2
;
582 tcg_ctx
.code_gen_buffer_size
= size1
;
587 #ifdef USE_STATIC_CODE_GEN_BUFFER
588 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
589 __attribute__((aligned(CODE_GEN_ALIGN
)));
592 static inline void do_protect(void *addr
, long size
, int prot
)
595 VirtualProtect(addr
, size
, prot
, &old_protect
);
598 static inline void map_exec(void *addr
, long size
)
600 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
603 static inline void map_none(void *addr
, long size
)
605 do_protect(addr
, size
, PAGE_NOACCESS
);
608 static inline void do_protect(void *addr
, long size
, int prot
)
610 uintptr_t start
, end
;
612 start
= (uintptr_t)addr
;
613 start
&= qemu_real_host_page_mask
;
615 end
= (uintptr_t)addr
+ size
;
616 end
= ROUND_UP(end
, qemu_real_host_page_size
);
618 mprotect((void *)start
, end
- start
, prot
);
621 static inline void map_exec(void *addr
, long size
)
623 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
626 static inline void map_none(void *addr
, long size
)
628 do_protect(addr
, size
, PROT_NONE
);
632 static inline void *alloc_code_gen_buffer(void)
634 void *buf
= static_code_gen_buffer
;
635 size_t full_size
, size
;
637 /* The size of the buffer, rounded down to end on a page boundary. */
638 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
639 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
641 /* Reserve a guard page. */
642 size
= full_size
- qemu_real_host_page_size
;
644 /* Honor a command-line option limiting the size of the buffer. */
645 if (size
> tcg_ctx
.code_gen_buffer_size
) {
646 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
647 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
649 tcg_ctx
.code_gen_buffer_size
= size
;
652 if (cross_256mb(buf
, size
)) {
653 buf
= split_cross_256mb(buf
, size
);
654 size
= tcg_ctx
.code_gen_buffer_size
;
659 map_none(buf
+ size
, qemu_real_host_page_size
);
660 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
664 #elif defined(_WIN32)
665 static inline void *alloc_code_gen_buffer(void)
667 size_t size
= tcg_ctx
.code_gen_buffer_size
;
670 /* Perform the allocation in two steps, so that the guard page
671 is reserved but uncommitted. */
672 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
673 MEM_RESERVE
, PAGE_NOACCESS
);
675 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
676 assert(buf1
== buf2
);
682 static inline void *alloc_code_gen_buffer(void)
684 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
686 size_t size
= tcg_ctx
.code_gen_buffer_size
;
689 /* Constrain the position of the buffer based on the host cpu.
690 Note that these addresses are chosen in concert with the
691 addresses assigned in the relevant linker script file. */
692 # if defined(__PIE__) || defined(__PIC__)
693 /* Don't bother setting a preferred location if we're building
694 a position-independent executable. We're more likely to get
695 an address near the main executable if we let the kernel
696 choose the address. */
697 # elif defined(__x86_64__) && defined(MAP_32BIT)
698 /* Force the memory down into low memory with the executable.
699 Leave the choice of exact location with the kernel. */
701 /* Cannot expect to map more than 800MB in low memory. */
702 if (size
> 800u * 1024 * 1024) {
703 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
705 # elif defined(__sparc__)
706 start
= 0x40000000ul
;
707 # elif defined(__s390x__)
708 start
= 0x90000000ul
;
709 # elif defined(__mips__)
710 # if _MIPS_SIM == _ABI64
711 start
= 0x128000000ul
;
713 start
= 0x08000000ul
;
717 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
718 PROT_NONE
, flags
, -1, 0);
719 if (buf
== MAP_FAILED
) {
724 if (cross_256mb(buf
, size
)) {
725 /* Try again, with the original still mapped, to avoid re-acquiring
726 that 256mb crossing. This time don't specify an address. */
728 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
729 PROT_NONE
, flags
, -1, 0);
730 switch ((int)(buf2
!= MAP_FAILED
)) {
732 if (!cross_256mb(buf2
, size
)) {
733 /* Success! Use the new buffer. */
734 munmap(buf
, size
+ qemu_real_host_page_size
);
737 /* Failure. Work with what we had. */
738 munmap(buf2
, size
+ qemu_real_host_page_size
);
741 /* Split the original buffer. Free the smaller half. */
742 buf2
= split_cross_256mb(buf
, size
);
743 size2
= tcg_ctx
.code_gen_buffer_size
;
745 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
747 munmap(buf
, size
- size2
);
756 /* Make the final buffer accessible. The guard page at the end
757 will remain inaccessible with PROT_NONE. */
758 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
760 /* Request large pages for the buffer. */
761 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
765 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
767 static inline void code_gen_alloc(size_t tb_size
)
769 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
770 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
771 if (tcg_ctx
.code_gen_buffer
== NULL
) {
772 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
776 /* size this conservatively -- realloc later if needed */
777 tcg_ctx
.tb_ctx
.tbs_size
=
778 tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
/ 8;
779 if (unlikely(!tcg_ctx
.tb_ctx
.tbs_size
)) {
780 tcg_ctx
.tb_ctx
.tbs_size
= 64 * 1024;
782 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
*, tcg_ctx
.tb_ctx
.tbs_size
);
784 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
787 static void tb_htable_init(void)
789 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
791 qht_init(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
, mode
);
794 /* Must be called before using the QEMU cpus. 'tb_size' is the size
795 (in bytes) allocated to the translation buffer. Zero means default
797 void tcg_exec_init(unsigned long tb_size
)
803 code_gen_alloc(tb_size
);
804 #if defined(CONFIG_SOFTMMU)
805 /* There's no guest base to take into account, so go ahead and
806 initialize the prologue now. */
807 tcg_prologue_init(&tcg_ctx
);
812 * Allocate a new translation block. Flush the translation buffer if
813 * too many translation blocks or too much generated code.
815 * Called with tb_lock held.
817 static TranslationBlock
*tb_alloc(target_ulong pc
)
819 TranslationBlock
*tb
;
824 tb
= tcg_tb_alloc(&tcg_ctx
);
825 if (unlikely(tb
== NULL
)) {
828 ctx
= &tcg_ctx
.tb_ctx
;
829 if (unlikely(ctx
->nb_tbs
== ctx
->tbs_size
)) {
831 ctx
->tbs
= g_renew(TranslationBlock
*, ctx
->tbs
, ctx
->tbs_size
);
833 ctx
->tbs
[ctx
->nb_tbs
++] = tb
;
837 /* Called with tb_lock held. */
838 void tb_free(TranslationBlock
*tb
)
842 /* In practice this is mostly used for single use temporary TB
843 Ignore the hard cases and just back up if this TB happens to
844 be the last one generated. */
845 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
846 tb
== tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
847 size_t struct_size
= ROUND_UP(sizeof(*tb
), qemu_icache_linesize
);
849 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
- struct_size
;
850 tcg_ctx
.tb_ctx
.nb_tbs
--;
854 static inline void invalidate_page_bitmap(PageDesc
*p
)
856 #ifdef CONFIG_SOFTMMU
857 g_free(p
->code_bitmap
);
858 p
->code_bitmap
= NULL
;
859 p
->code_write_count
= 0;
863 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
864 static void page_flush_tb_1(int level
, void **lp
)
874 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
875 pd
[i
].first_tb
= NULL
;
876 invalidate_page_bitmap(pd
+ i
);
881 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
882 page_flush_tb_1(level
- 1, pp
+ i
);
887 static void page_flush_tb(void)
889 int i
, l1_sz
= v_l1_size
;
891 for (i
= 0; i
< l1_sz
; i
++) {
892 page_flush_tb_1(v_l2_levels
, l1_map
+ i
);
896 /* flush all the translation blocks */
897 static void do_tb_flush(CPUState
*cpu
, run_on_cpu_data tb_flush_count
)
901 /* If it is already been done on request of another CPU,
904 if (tcg_ctx
.tb_ctx
.tb_flush_count
!= tb_flush_count
.host_int
) {
908 if (DEBUG_TB_FLUSH_GATE
) {
909 printf("qemu: flush code_size=%td nb_tbs=%d avg_tb_size=%td\n",
910 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
911 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
912 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) /
913 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
915 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
916 > tcg_ctx
.code_gen_buffer_size
) {
917 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
921 cpu_tb_jmp_cache_clear(cpu
);
924 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
925 qht_reset_size(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
928 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
929 /* XXX: flush processor icache at this point if cache flush is
931 atomic_mb_set(&tcg_ctx
.tb_ctx
.tb_flush_count
,
932 tcg_ctx
.tb_ctx
.tb_flush_count
+ 1);
938 void tb_flush(CPUState
*cpu
)
941 unsigned tb_flush_count
= atomic_mb_read(&tcg_ctx
.tb_ctx
.tb_flush_count
);
942 async_safe_run_on_cpu(cpu
, do_tb_flush
,
943 RUN_ON_CPU_HOST_INT(tb_flush_count
));
947 #ifdef DEBUG_TB_CHECK
950 do_tb_invalidate_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
952 TranslationBlock
*tb
= p
;
953 target_ulong addr
= *(target_ulong
*)userp
;
955 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
956 printf("ERROR invalidate: address=" TARGET_FMT_lx
957 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
961 /* verify that all the pages have correct rights for code
963 * Called with tb_lock held.
965 static void tb_invalidate_check(target_ulong address
)
967 address
&= TARGET_PAGE_MASK
;
968 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
972 do_tb_page_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
974 TranslationBlock
*tb
= p
;
977 flags1
= page_get_flags(tb
->pc
);
978 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
979 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
980 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
981 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
985 /* verify that all the pages have correct rights for code */
986 static void tb_page_check(void)
988 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_page_check
, NULL
);
993 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
995 TranslationBlock
*tb1
;
1000 n1
= (uintptr_t)tb1
& 3;
1001 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
1003 *ptb
= tb1
->page_next
[n1
];
1006 ptb
= &tb1
->page_next
[n1
];
1010 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
1011 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
1013 TranslationBlock
*tb1
;
1014 uintptr_t *ptb
, ntb
;
1017 ptb
= &tb
->jmp_list_next
[n
];
1019 /* find tb(n) in circular list */
1023 tb1
= (TranslationBlock
*)(ntb
& ~3);
1024 if (n1
== n
&& tb1
== tb
) {
1028 ptb
= &tb1
->jmp_list_first
;
1030 ptb
= &tb1
->jmp_list_next
[n1
];
1033 /* now we can suppress tb(n) from the list */
1034 *ptb
= tb
->jmp_list_next
[n
];
1036 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
1040 /* reset the jump entry 'n' of a TB so that it is not chained to
1042 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
1044 uintptr_t addr
= (uintptr_t)(tb
->tc_ptr
+ tb
->jmp_reset_offset
[n
]);
1045 tb_set_jmp_target(tb
, n
, addr
);
1048 /* remove any jumps to the TB */
1049 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
1051 TranslationBlock
*tb1
;
1052 uintptr_t *ptb
, ntb
;
1055 ptb
= &tb
->jmp_list_first
;
1059 tb1
= (TranslationBlock
*)(ntb
& ~3);
1063 tb_reset_jump(tb1
, n1
);
1064 *ptb
= tb1
->jmp_list_next
[n1
];
1065 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
1069 /* invalidate one TB
1071 * Called with tb_lock held.
1073 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
1078 tb_page_addr_t phys_pc
;
1082 atomic_set(&tb
->cflags
, tb
->cflags
| CF_INVALID
);
1084 /* remove the TB from the hash list */
1085 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1086 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->trace_vcpu_dstate
);
1087 qht_remove(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1089 /* remove the TB from the page list */
1090 if (tb
->page_addr
[0] != page_addr
) {
1091 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1092 tb_page_remove(&p
->first_tb
, tb
);
1093 invalidate_page_bitmap(p
);
1095 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1096 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1097 tb_page_remove(&p
->first_tb
, tb
);
1098 invalidate_page_bitmap(p
);
1101 /* remove the TB from the hash list */
1102 h
= tb_jmp_cache_hash_func(tb
->pc
);
1104 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1105 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1109 /* suppress this TB from the two jump lists */
1110 tb_remove_from_jmp_list(tb
, 0);
1111 tb_remove_from_jmp_list(tb
, 1);
1113 /* suppress any remaining jumps to this TB */
1116 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1119 #ifdef CONFIG_SOFTMMU
1120 static void build_page_bitmap(PageDesc
*p
)
1122 int n
, tb_start
, tb_end
;
1123 TranslationBlock
*tb
;
1125 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1128 while (tb
!= NULL
) {
1129 n
= (uintptr_t)tb
& 3;
1130 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1131 /* NOTE: this is subtle as a TB may span two physical pages */
1133 /* NOTE: tb_end may be after the end of the page, but
1134 it is not a problem */
1135 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1136 tb_end
= tb_start
+ tb
->size
;
1137 if (tb_end
> TARGET_PAGE_SIZE
) {
1138 tb_end
= TARGET_PAGE_SIZE
;
1142 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1144 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1145 tb
= tb
->page_next
[n
];
1150 /* add the tb in the target page and protect it if necessary
1152 * Called with mmap_lock held for user-mode emulation.
1154 static inline void tb_alloc_page(TranslationBlock
*tb
,
1155 unsigned int n
, tb_page_addr_t page_addr
)
1158 #ifndef CONFIG_USER_ONLY
1159 bool page_already_protected
;
1162 assert_memory_lock();
1164 tb
->page_addr
[n
] = page_addr
;
1165 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1166 tb
->page_next
[n
] = p
->first_tb
;
1167 #ifndef CONFIG_USER_ONLY
1168 page_already_protected
= p
->first_tb
!= NULL
;
1170 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1171 invalidate_page_bitmap(p
);
1173 #if defined(CONFIG_USER_ONLY)
1174 if (p
->flags
& PAGE_WRITE
) {
1179 /* force the host page as non writable (writes will have a
1180 page fault + mprotect overhead) */
1181 page_addr
&= qemu_host_page_mask
;
1183 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1184 addr
+= TARGET_PAGE_SIZE
) {
1186 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1191 p2
->flags
&= ~PAGE_WRITE
;
1193 mprotect(g2h(page_addr
), qemu_host_page_size
,
1194 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1195 #ifdef DEBUG_TB_INVALIDATE
1196 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1201 /* if some code is already present, then the pages are already
1202 protected. So we handle the case where only the first TB is
1203 allocated in a physical page */
1204 if (!page_already_protected
) {
1205 tlb_protect_code(page_addr
);
1210 /* add a new TB and link it to the physical page tables. phys_page2 is
1211 * (-1) to indicate that only one page contains the TB.
1213 * Called with mmap_lock held for user-mode emulation.
1215 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1216 tb_page_addr_t phys_page2
)
1220 assert_memory_lock();
1222 /* add in the page list */
1223 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1224 if (phys_page2
!= -1) {
1225 tb_alloc_page(tb
, 1, phys_page2
);
1227 tb
->page_addr
[1] = -1;
1230 /* add in the hash table */
1231 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->trace_vcpu_dstate
);
1232 qht_insert(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1234 #ifdef DEBUG_TB_CHECK
1239 /* Called with mmap_lock held for user mode emulation. */
1240 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1241 target_ulong pc
, target_ulong cs_base
,
1242 uint32_t flags
, int cflags
)
1244 CPUArchState
*env
= cpu
->env_ptr
;
1245 TranslationBlock
*tb
;
1246 tb_page_addr_t phys_pc
, phys_page2
;
1247 target_ulong virt_page2
;
1248 tcg_insn_unit
*gen_code_buf
;
1249 int gen_code_size
, search_size
;
1250 #ifdef CONFIG_PROFILER
1253 assert_memory_lock();
1255 phys_pc
= get_page_addr_code(env
, pc
);
1256 if (use_icount
&& !(cflags
& CF_IGNORE_ICOUNT
)) {
1257 cflags
|= CF_USE_ICOUNT
;
1261 if (unlikely(!tb
)) {
1263 /* flush must be done */
1266 /* Make the execution loop process the flush as soon as possible. */
1267 cpu
->exception_index
= EXCP_INTERRUPT
;
1271 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1272 tb
->tc_ptr
= gen_code_buf
;
1274 tb
->cs_base
= cs_base
;
1276 tb
->cflags
= cflags
;
1277 tb
->trace_vcpu_dstate
= *cpu
->trace_dstate
;
1279 #ifdef CONFIG_PROFILER
1280 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1282 ti
= profile_getclock();
1285 tcg_func_start(&tcg_ctx
);
1287 tcg_ctx
.cpu
= ENV_GET_CPU(env
);
1288 gen_intermediate_code(cpu
, tb
);
1291 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
1293 /* generate machine code */
1294 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1295 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1296 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1297 if (TCG_TARGET_HAS_direct_jump
) {
1298 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_target_arg
;
1299 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1301 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1302 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_arg
;
1305 #ifdef CONFIG_PROFILER
1307 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1308 ti
= profile_getclock();
1311 /* ??? Overflow could be handled better here. In particular, we
1312 don't need to re-do gen_intermediate_code, nor should we re-do
1313 the tcg optimization currently hidden inside tcg_gen_code. All
1314 that should be required is to flush the TBs, allocate a new TB,
1315 re-initialize it per above, and re-do the actual code generation. */
1316 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1317 if (unlikely(gen_code_size
< 0)) {
1318 goto buffer_overflow
;
1320 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1321 if (unlikely(search_size
< 0)) {
1322 goto buffer_overflow
;
1325 #ifdef CONFIG_PROFILER
1326 tcg_ctx
.code_time
+= profile_getclock() - ti
;
1327 tcg_ctx
.code_in_len
+= tb
->size
;
1328 tcg_ctx
.code_out_len
+= gen_code_size
;
1329 tcg_ctx
.search_out_len
+= search_size
;
1333 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1334 qemu_log_in_addr_range(tb
->pc
)) {
1336 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1337 if (tcg_ctx
.data_gen_ptr
) {
1338 size_t code_size
= tcg_ctx
.data_gen_ptr
- tb
->tc_ptr
;
1339 size_t data_size
= gen_code_size
- code_size
;
1342 log_disas(tb
->tc_ptr
, code_size
);
1344 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1345 if (sizeof(tcg_target_ulong
) == 8) {
1346 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1347 (uintptr_t)tcg_ctx
.data_gen_ptr
+ i
,
1348 *(uint64_t *)(tcg_ctx
.data_gen_ptr
+ i
));
1350 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1351 (uintptr_t)tcg_ctx
.data_gen_ptr
+ i
,
1352 *(uint32_t *)(tcg_ctx
.data_gen_ptr
+ i
));
1356 log_disas(tb
->tc_ptr
, gen_code_size
);
1364 tcg_ctx
.code_gen_ptr
= (void *)
1365 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1368 /* init jump list */
1369 assert(((uintptr_t)tb
& 3) == 0);
1370 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1371 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1372 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1374 /* init original jump addresses wich has been set during tcg_gen_code() */
1375 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1376 tb_reset_jump(tb
, 0);
1378 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1379 tb_reset_jump(tb
, 1);
1382 /* check next page if needed */
1383 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1385 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1386 phys_page2
= get_page_addr_code(env
, virt_page2
);
1388 /* As long as consistency of the TB stuff is provided by tb_lock in user
1389 * mode and is implicit in single-threaded softmmu emulation, no explicit
1390 * memory barrier is required before tb_link_page() makes the TB visible
1391 * through the physical hash table and physical page list.
1393 tb_link_page(tb
, phys_pc
, phys_page2
);
1398 * Invalidate all TBs which intersect with the target physical address range
1399 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1400 * 'is_cpu_write_access' should be true if called from a real cpu write
1401 * access: the virtual CPU will exit the current TB if code is modified inside
1404 * Called with mmap_lock held for user-mode emulation, grabs tb_lock
1405 * Called with tb_lock held for system-mode emulation
1407 static void tb_invalidate_phys_range_1(tb_page_addr_t start
, tb_page_addr_t end
)
1409 while (start
< end
) {
1410 tb_invalidate_phys_page_range(start
, end
, 0);
1411 start
&= TARGET_PAGE_MASK
;
1412 start
+= TARGET_PAGE_SIZE
;
1416 #ifdef CONFIG_SOFTMMU
1417 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1420 tb_invalidate_phys_range_1(start
, end
);
1423 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1425 assert_memory_lock();
1427 tb_invalidate_phys_range_1(start
, end
);
1432 * Invalidate all TBs which intersect with the target physical address range
1433 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1434 * 'is_cpu_write_access' should be true if called from a real cpu write
1435 * access: the virtual CPU will exit the current TB if code is modified inside
1438 * Called with tb_lock/mmap_lock held for user-mode emulation
1439 * Called with tb_lock held for system-mode emulation
1441 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1442 int is_cpu_write_access
)
1444 TranslationBlock
*tb
, *tb_next
;
1445 #if defined(TARGET_HAS_PRECISE_SMC)
1446 CPUState
*cpu
= current_cpu
;
1447 CPUArchState
*env
= NULL
;
1449 tb_page_addr_t tb_start
, tb_end
;
1452 #ifdef TARGET_HAS_PRECISE_SMC
1453 int current_tb_not_found
= is_cpu_write_access
;
1454 TranslationBlock
*current_tb
= NULL
;
1455 int current_tb_modified
= 0;
1456 target_ulong current_pc
= 0;
1457 target_ulong current_cs_base
= 0;
1458 uint32_t current_flags
= 0;
1459 #endif /* TARGET_HAS_PRECISE_SMC */
1461 assert_memory_lock();
1464 p
= page_find(start
>> TARGET_PAGE_BITS
);
1468 #if defined(TARGET_HAS_PRECISE_SMC)
1474 /* we remove all the TBs in the range [start, end[ */
1475 /* XXX: see if in some cases it could be faster to invalidate all
1478 while (tb
!= NULL
) {
1479 n
= (uintptr_t)tb
& 3;
1480 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1481 tb_next
= tb
->page_next
[n
];
1482 /* NOTE: this is subtle as a TB may span two physical pages */
1484 /* NOTE: tb_end may be after the end of the page, but
1485 it is not a problem */
1486 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1487 tb_end
= tb_start
+ tb
->size
;
1489 tb_start
= tb
->page_addr
[1];
1490 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1492 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1493 #ifdef TARGET_HAS_PRECISE_SMC
1494 if (current_tb_not_found
) {
1495 current_tb_not_found
= 0;
1497 if (cpu
->mem_io_pc
) {
1498 /* now we have a real cpu fault */
1499 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1502 if (current_tb
== tb
&&
1503 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1504 /* If we are modifying the current TB, we must stop
1505 its execution. We could be more precise by checking
1506 that the modification is after the current PC, but it
1507 would require a specialized function to partially
1508 restore the CPU state */
1510 current_tb_modified
= 1;
1511 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1512 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1515 #endif /* TARGET_HAS_PRECISE_SMC */
1516 tb_phys_invalidate(tb
, -1);
1520 #if !defined(CONFIG_USER_ONLY)
1521 /* if no code remaining, no need to continue to use slow writes */
1523 invalidate_page_bitmap(p
);
1524 tlb_unprotect_code(start
);
1527 #ifdef TARGET_HAS_PRECISE_SMC
1528 if (current_tb_modified
) {
1529 /* we generate a block containing just the instruction
1530 modifying the memory. It will ensure that it cannot modify
1532 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1533 cpu_loop_exit_noexc(cpu
);
1538 #ifdef CONFIG_SOFTMMU
1539 /* len must be <= 8 and start must be a multiple of len.
1540 * Called via softmmu_template.h when code areas are written to with
1541 * iothread mutex not held.
1543 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1549 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1550 cpu_single_env
->mem_io_vaddr
, len
,
1551 cpu_single_env
->eip
,
1552 cpu_single_env
->eip
+
1553 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1556 assert_memory_lock();
1558 p
= page_find(start
>> TARGET_PAGE_BITS
);
1562 if (!p
->code_bitmap
&&
1563 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1564 /* build code bitmap. FIXME: writes should be protected by
1565 * tb_lock, reads by tb_lock or RCU.
1567 build_page_bitmap(p
);
1569 if (p
->code_bitmap
) {
1573 nr
= start
& ~TARGET_PAGE_MASK
;
1574 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1575 if (b
& ((1 << len
) - 1)) {
1580 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1584 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1585 * host PC of the faulting store instruction that caused this invalidate.
1586 * Returns true if the caller needs to abort execution of the current
1587 * TB (because it was modified by this store and the guest CPU has
1588 * precise-SMC semantics).
1590 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
1592 TranslationBlock
*tb
;
1595 #ifdef TARGET_HAS_PRECISE_SMC
1596 TranslationBlock
*current_tb
= NULL
;
1597 CPUState
*cpu
= current_cpu
;
1598 CPUArchState
*env
= NULL
;
1599 int current_tb_modified
= 0;
1600 target_ulong current_pc
= 0;
1601 target_ulong current_cs_base
= 0;
1602 uint32_t current_flags
= 0;
1605 assert_memory_lock();
1607 addr
&= TARGET_PAGE_MASK
;
1608 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1615 #ifdef TARGET_HAS_PRECISE_SMC
1616 if (tb
&& pc
!= 0) {
1617 current_tb
= tb_find_pc(pc
);
1623 while (tb
!= NULL
) {
1624 n
= (uintptr_t)tb
& 3;
1625 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1626 #ifdef TARGET_HAS_PRECISE_SMC
1627 if (current_tb
== tb
&&
1628 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1629 /* If we are modifying the current TB, we must stop
1630 its execution. We could be more precise by checking
1631 that the modification is after the current PC, but it
1632 would require a specialized function to partially
1633 restore the CPU state */
1635 current_tb_modified
= 1;
1636 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1637 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1640 #endif /* TARGET_HAS_PRECISE_SMC */
1641 tb_phys_invalidate(tb
, addr
);
1642 tb
= tb
->page_next
[n
];
1645 #ifdef TARGET_HAS_PRECISE_SMC
1646 if (current_tb_modified
) {
1647 /* we generate a block containing just the instruction
1648 modifying the memory. It will ensure that it cannot modify
1650 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1651 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps
1652 * back into the cpu_exec loop. */
1662 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1663 tb[1].tc_ptr. Return NULL if not found */
1664 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1666 int m_min
, m_max
, m
;
1668 TranslationBlock
*tb
;
1670 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1673 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1674 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1677 /* binary search (cf Knuth) */
1679 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1680 while (m_min
<= m_max
) {
1681 m
= (m_min
+ m_max
) >> 1;
1682 tb
= tcg_ctx
.tb_ctx
.tbs
[m
];
1683 v
= (uintptr_t)tb
->tc_ptr
;
1686 } else if (tc_ptr
< v
) {
1692 return tcg_ctx
.tb_ctx
.tbs
[m_max
];
1695 #if !defined(CONFIG_USER_ONLY)
1696 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1698 ram_addr_t ram_addr
;
1703 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1704 if (!(memory_region_is_ram(mr
)
1705 || memory_region_is_romd(mr
))) {
1709 ram_addr
= memory_region_get_ram_addr(mr
) + addr
;
1711 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1715 #endif /* !defined(CONFIG_USER_ONLY) */
1717 /* Called with tb_lock held. */
1718 void tb_check_watchpoint(CPUState
*cpu
)
1720 TranslationBlock
*tb
;
1722 tb
= tb_find_pc(cpu
->mem_io_pc
);
1724 /* We can use retranslation to find the PC. */
1725 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1726 tb_phys_invalidate(tb
, -1);
1728 /* The exception probably happened in a helper. The CPU state should
1729 have been saved before calling it. Fetch the PC from there. */
1730 CPUArchState
*env
= cpu
->env_ptr
;
1731 target_ulong pc
, cs_base
;
1732 tb_page_addr_t addr
;
1735 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1736 addr
= get_page_addr_code(env
, pc
);
1737 tb_invalidate_phys_range(addr
, addr
+ 1);
1741 #ifndef CONFIG_USER_ONLY
1742 /* in deterministic execution mode, instructions doing device I/Os
1743 * must be at the end of the TB.
1745 * Called by softmmu_template.h, with iothread mutex not held.
1747 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1749 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1750 CPUArchState
*env
= cpu
->env_ptr
;
1752 TranslationBlock
*tb
;
1754 target_ulong pc
, cs_base
;
1758 tb
= tb_find_pc(retaddr
);
1760 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1763 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1764 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1765 /* Calculate how many instructions had been executed before the fault
1767 n
= n
- cpu
->icount_decr
.u16
.low
;
1768 /* Generate a new TB ending on the I/O insn. */
1770 /* On MIPS and SH, delay slot instructions can only be restarted if
1771 they were already the first instruction in the TB. If this is not
1772 the first instruction in a TB then re-execute the preceding
1774 #if defined(TARGET_MIPS)
1775 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1776 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1777 cpu
->icount_decr
.u16
.low
++;
1778 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1780 #elif defined(TARGET_SH4)
1781 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1784 cpu
->icount_decr
.u16
.low
++;
1785 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1788 /* This should never happen. */
1789 if (n
> CF_COUNT_MASK
) {
1790 cpu_abort(cpu
, "TB too big during recompile");
1793 cflags
= n
| CF_LAST_IO
;
1795 cs_base
= tb
->cs_base
;
1797 tb_phys_invalidate(tb
, -1);
1798 if (tb
->cflags
& CF_NOCACHE
) {
1800 /* Invalidate original TB if this TB was generated in
1801 * cpu_exec_nocache() */
1802 tb_phys_invalidate(tb
->orig_tb
, -1);
1806 /* FIXME: In theory this could raise an exception. In practice
1807 we have already translated the block once so it's probably ok. */
1808 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1810 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1811 * the first in the TB) then we end up generating a whole new TB and
1812 * repeating the fault, which is horribly inefficient.
1813 * Better would be to execute just this insn uncached, or generate a
1816 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the
1817 * tb_lock gets reset.
1819 cpu_loop_exit_noexc(cpu
);
1822 static void tb_jmp_cache_clear_page(CPUState
*cpu
, target_ulong page_addr
)
1824 unsigned int i
, i0
= tb_jmp_cache_hash_page(page_addr
);
1826 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
1827 atomic_set(&cpu
->tb_jmp_cache
[i0
+ i
], NULL
);
1831 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1833 /* Discard jump cache entries for any tb which might potentially
1834 overlap the flushed page. */
1835 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
1836 tb_jmp_cache_clear_page(cpu
, addr
);
1839 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
1840 struct qht_stats hst
)
1842 uint32_t hgram_opts
;
1846 if (!hst
.head_buckets
) {
1849 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1850 hst
.used_head_buckets
, hst
.head_buckets
,
1851 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
1853 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1854 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
1855 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
1856 hgram_opts
|= QDIST_PR_NODECIMAL
;
1858 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
1859 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1860 qdist_avg(&hst
.occupancy
) * 100, hgram
);
1863 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1864 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
1865 if (hgram_bins
> 10) {
1869 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
1871 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
1872 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1873 qdist_avg(&hst
.chain
), hgram
);
1877 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1879 int i
, target_code_size
, max_target_code_size
;
1880 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1881 TranslationBlock
*tb
;
1882 struct qht_stats hst
;
1886 target_code_size
= 0;
1887 max_target_code_size
= 0;
1889 direct_jmp_count
= 0;
1890 direct_jmp2_count
= 0;
1891 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1892 tb
= tcg_ctx
.tb_ctx
.tbs
[i
];
1893 target_code_size
+= tb
->size
;
1894 if (tb
->size
> max_target_code_size
) {
1895 max_target_code_size
= tb
->size
;
1897 if (tb
->page_addr
[1] != -1) {
1900 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1902 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1903 direct_jmp2_count
++;
1907 /* XXX: avoid using doubles ? */
1908 cpu_fprintf(f
, "Translation buffer state:\n");
1909 cpu_fprintf(f
, "gen code size %td/%zd\n",
1910 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1911 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1912 cpu_fprintf(f
, "TB count %d\n", tcg_ctx
.tb_ctx
.nb_tbs
);
1913 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1914 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1915 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1916 max_target_code_size
);
1917 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1918 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1919 tcg_ctx
.code_gen_buffer
) /
1920 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1921 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1922 tcg_ctx
.code_gen_buffer
) /
1923 target_code_size
: 0);
1924 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1925 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1926 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1927 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1929 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1930 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1932 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1933 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1935 qht_statistics_init(&tcg_ctx
.tb_ctx
.htable
, &hst
);
1936 print_qht_statistics(f
, cpu_fprintf
, hst
);
1937 qht_statistics_destroy(&hst
);
1939 cpu_fprintf(f
, "\nStatistics:\n");
1940 cpu_fprintf(f
, "TB flush count %u\n",
1941 atomic_read(&tcg_ctx
.tb_ctx
.tb_flush_count
));
1942 cpu_fprintf(f
, "TB invalidate count %d\n",
1943 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1944 cpu_fprintf(f
, "TLB flush count %zu\n", tlb_flush_count());
1945 tcg_dump_info(f
, cpu_fprintf
);
1950 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1952 tcg_dump_op_count(f
, cpu_fprintf
);
1955 #else /* CONFIG_USER_ONLY */
1957 void cpu_interrupt(CPUState
*cpu
, int mask
)
1959 g_assert(qemu_mutex_iothread_locked());
1960 cpu
->interrupt_request
|= mask
;
1961 cpu
->icount_decr
.u16
.high
= -1;
1965 * Walks guest process memory "regions" one by one
1966 * and calls callback function 'fn' for each region.
1968 struct walk_memory_regions_data
{
1969 walk_memory_regions_fn fn
;
1975 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1976 target_ulong end
, int new_prot
)
1978 if (data
->start
!= -1u) {
1979 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1985 data
->start
= (new_prot
? end
: -1u);
1986 data
->prot
= new_prot
;
1991 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1992 target_ulong base
, int level
, void **lp
)
1998 return walk_memory_regions_end(data
, base
, 0);
2004 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2005 int prot
= pd
[i
].flags
;
2007 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2008 if (prot
!= data
->prot
) {
2009 rc
= walk_memory_regions_end(data
, pa
, prot
);
2018 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2019 pa
= base
| ((target_ulong
)i
<<
2020 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
2021 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2031 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2033 struct walk_memory_regions_data data
;
2034 uintptr_t i
, l1_sz
= v_l1_size
;
2041 for (i
= 0; i
< l1_sz
; i
++) {
2042 target_ulong base
= i
<< (v_l1_shift
+ TARGET_PAGE_BITS
);
2043 int rc
= walk_memory_regions_1(&data
, base
, v_l2_levels
, l1_map
+ i
);
2049 return walk_memory_regions_end(&data
, 0, 0);
2052 static int dump_region(void *priv
, target_ulong start
,
2053 target_ulong end
, unsigned long prot
)
2055 FILE *f
= (FILE *)priv
;
2057 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
2058 " "TARGET_FMT_lx
" %c%c%c\n",
2059 start
, end
, end
- start
,
2060 ((prot
& PAGE_READ
) ? 'r' : '-'),
2061 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2062 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2067 /* dump memory mappings */
2068 void page_dump(FILE *f
)
2070 const int length
= sizeof(target_ulong
) * 2;
2071 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
2072 length
, "start", length
, "end", length
, "size", "prot");
2073 walk_memory_regions(f
, dump_region
);
2076 int page_get_flags(target_ulong address
)
2080 p
= page_find(address
>> TARGET_PAGE_BITS
);
2087 /* Modify the flags of a page and invalidate the code if necessary.
2088 The flag PAGE_WRITE_ORG is positioned automatically depending
2089 on PAGE_WRITE. The mmap_lock should already be held. */
2090 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2092 target_ulong addr
, len
;
2094 /* This function should never be called with addresses outside the
2095 guest address space. If this assert fires, it probably indicates
2096 a missing call to h2g_valid. */
2097 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2098 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2100 assert(start
< end
);
2101 assert_memory_lock();
2103 start
= start
& TARGET_PAGE_MASK
;
2104 end
= TARGET_PAGE_ALIGN(end
);
2106 if (flags
& PAGE_WRITE
) {
2107 flags
|= PAGE_WRITE_ORG
;
2110 for (addr
= start
, len
= end
- start
;
2112 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2113 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2115 /* If the write protection bit is set, then we invalidate
2117 if (!(p
->flags
& PAGE_WRITE
) &&
2118 (flags
& PAGE_WRITE
) &&
2120 tb_invalidate_phys_page(addr
, 0);
2126 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2132 /* This function should never be called with addresses outside the
2133 guest address space. If this assert fires, it probably indicates
2134 a missing call to h2g_valid. */
2135 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2136 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2142 if (start
+ len
- 1 < start
) {
2143 /* We've wrapped around. */
2147 /* must do before we loose bits in the next step */
2148 end
= TARGET_PAGE_ALIGN(start
+ len
);
2149 start
= start
& TARGET_PAGE_MASK
;
2151 for (addr
= start
, len
= end
- start
;
2153 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2154 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2158 if (!(p
->flags
& PAGE_VALID
)) {
2162 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
2165 if (flags
& PAGE_WRITE
) {
2166 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2169 /* unprotect the page if it was put read-only because it
2170 contains translated code */
2171 if (!(p
->flags
& PAGE_WRITE
)) {
2172 if (!page_unprotect(addr
, 0)) {
2181 /* called from signal handler: invalidate the code and unprotect the
2182 * page. Return 0 if the fault was not handled, 1 if it was handled,
2183 * and 2 if it was handled but the caller must cause the TB to be
2184 * immediately exited. (We can only return 2 if the 'pc' argument is
2187 int page_unprotect(target_ulong address
, uintptr_t pc
)
2190 bool current_tb_invalidated
;
2192 target_ulong host_start
, host_end
, addr
;
2194 /* Technically this isn't safe inside a signal handler. However we
2195 know this only ever happens in a synchronous SEGV handler, so in
2196 practice it seems to be ok. */
2199 p
= page_find(address
>> TARGET_PAGE_BITS
);
2205 /* if the page was really writable, then we change its
2206 protection back to writable */
2207 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2208 host_start
= address
& qemu_host_page_mask
;
2209 host_end
= host_start
+ qemu_host_page_size
;
2212 current_tb_invalidated
= false;
2213 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2214 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2215 p
->flags
|= PAGE_WRITE
;
2218 /* and since the content will be modified, we must invalidate
2219 the corresponding translated code. */
2220 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2221 #ifdef DEBUG_TB_CHECK
2222 tb_invalidate_check(addr
);
2225 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2229 /* If current TB was invalidated return to main loop */
2230 return current_tb_invalidated
? 2 : 1;
2235 #endif /* CONFIG_USER_ONLY */
2237 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2238 void tcg_flush_softmmu_tlb(CPUState
*cs
)
2240 #ifdef CONFIG_SOFTMMU