4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
22 #include "qemu/osdep.h"
25 #include "qemu-common.h"
26 #define NO_CPU_IO_DEFS
29 #include "disas/disas.h"
30 #include "exec/exec-all.h"
32 #if defined(CONFIG_USER_ONLY)
34 #include "exec/exec-all.h"
35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
36 #include <sys/param.h>
37 #if __FreeBSD_version >= 700104
38 #define HAVE_KINFO_GETVMMAP
39 #define sigqueue sigqueue_freebsd /* avoid redefinition */
41 #include <machine/profile.h>
50 #include "exec/address-spaces.h"
53 #include "exec/cputlb.h"
54 #include "exec/tb-hash.h"
55 #include "translate-all.h"
56 #include "qemu/bitmap.h"
57 #include "qemu/error-report.h"
58 #include "qemu/timer.h"
59 #include "qemu/main-loop.h"
61 #include "sysemu/cpus.h"
63 /* #define DEBUG_TB_INVALIDATE */
64 /* #define DEBUG_TB_FLUSH */
65 /* make various TB consistency checks */
66 /* #define DEBUG_TB_CHECK */
68 #ifdef DEBUG_TB_INVALIDATE
69 #define DEBUG_TB_INVALIDATE_GATE 1
71 #define DEBUG_TB_INVALIDATE_GATE 0
75 #define DEBUG_TB_FLUSH_GATE 1
77 #define DEBUG_TB_FLUSH_GATE 0
80 #if !defined(CONFIG_USER_ONLY)
81 /* TB consistency checks only implemented for usermode emulation. */
86 #define DEBUG_TB_CHECK_GATE 1
88 #define DEBUG_TB_CHECK_GATE 0
91 /* Access to the various translations structures need to be serialised via locks
92 * for consistency. This is automatic for SoftMMU based system
93 * emulation due to its single threaded nature. In user-mode emulation
94 * access to the memory related structures are protected with the
98 #define assert_memory_lock() tcg_debug_assert(have_tb_lock)
100 #define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
103 #define SMC_BITMAP_USE_THRESHOLD 10
105 typedef struct PageDesc
{
106 /* list of TBs intersecting this ram page */
107 TranslationBlock
*first_tb
;
108 #ifdef CONFIG_SOFTMMU
109 /* in order to optimize self modifying code, we count the number
110 of lookups we do to a given page to use a bitmap */
111 unsigned int code_write_count
;
112 unsigned long *code_bitmap
;
118 /* In system mode we want L1_MAP to be based on ram offsets,
119 while in user mode we want it to be based on virtual addresses. */
120 #if !defined(CONFIG_USER_ONLY)
121 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
122 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
124 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
127 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
130 /* Size of the L2 (and L3, etc) page tables. */
132 #define V_L2_SIZE (1 << V_L2_BITS)
134 /* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
135 QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS
>
136 sizeof(((TranslationBlock
*)0)->trace_vcpu_dstate
)
140 * L1 Mapping properties
142 static int v_l1_size
;
143 static int v_l1_shift
;
144 static int v_l2_levels
;
146 /* The bottom level has pointers to PageDesc, and is indexed by
147 * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
149 #define V_L1_MIN_BITS 4
150 #define V_L1_MAX_BITS (V_L2_BITS + 3)
151 #define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
153 static void *l1_map
[V_L1_MAX_SIZE
];
155 /* code generation context */
159 /* translation block context */
160 static __thread
int have_tb_lock
;
162 static void page_table_config_init(void)
166 assert(TARGET_PAGE_BITS
);
167 /* The bits remaining after N lower levels of page tables. */
168 v_l1_bits
= (L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
) % V_L2_BITS
;
169 if (v_l1_bits
< V_L1_MIN_BITS
) {
170 v_l1_bits
+= V_L2_BITS
;
173 v_l1_size
= 1 << v_l1_bits
;
174 v_l1_shift
= L1_MAP_ADDR_SPACE_BITS
- TARGET_PAGE_BITS
- v_l1_bits
;
175 v_l2_levels
= v_l1_shift
/ V_L2_BITS
- 1;
177 assert(v_l1_bits
<= V_L1_MAX_BITS
);
178 assert(v_l1_shift
% V_L2_BITS
== 0);
179 assert(v_l2_levels
>= 0);
182 #define assert_tb_locked() tcg_debug_assert(have_tb_lock)
183 #define assert_tb_unlocked() tcg_debug_assert(!have_tb_lock)
187 assert_tb_unlocked();
188 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
196 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
199 void tb_lock_reset(void)
202 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
207 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
209 void cpu_gen_init(void)
211 tcg_context_init(&tcg_ctx
);
214 /* Encode VAL as a signed leb128 sequence at P.
215 Return P incremented past the encoded value. */
216 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
223 more
= !((val
== 0 && (byte
& 0x40) == 0)
224 || (val
== -1 && (byte
& 0x40) != 0));
234 /* Decode a signed leb128 sequence at *PP; increment *PP past the
235 decoded value. Return the decoded value. */
236 static target_long
decode_sleb128(uint8_t **pp
)
244 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
246 } while (byte
& 0x80);
247 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
248 val
|= -(target_ulong
)1 << shift
;
255 /* Encode the data collected about the instructions while compiling TB.
256 Place the data at BLOCK, and return the number of bytes consumed.
258 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
259 which come from the target's insn_start data, followed by a uintptr_t
260 which comes from the host pc of the end of the code implementing the insn.
262 Each line of the table is encoded as sleb128 deltas from the previous
263 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }.
264 That is, the first column is seeded with the guest pc, the last column
265 with the host pc, and the middle columns with zeros. */
267 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
269 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
273 tb
->tc
.search
= block
;
275 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
278 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
280 prev
= (j
== 0 ? tb
->pc
: 0);
282 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
284 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
286 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
287 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
289 /* Test for (pending) buffer overflow. The assumption is that any
290 one row beginning below the high water mark cannot overrun
291 the buffer completely. Thus we can test for overflow after
292 encoding a row without having to check during encoding. */
293 if (unlikely(p
> highwater
)) {
301 /* The cpu state corresponding to 'searched_pc' is restored.
302 * Called with tb_lock held.
304 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
305 uintptr_t searched_pc
)
307 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
308 uintptr_t host_pc
= (uintptr_t)tb
->tc
.ptr
;
309 CPUArchState
*env
= cpu
->env_ptr
;
310 uint8_t *p
= tb
->tc
.search
;
311 int i
, j
, num_insns
= tb
->icount
;
312 #ifdef CONFIG_PROFILER
313 int64_t ti
= profile_getclock();
316 searched_pc
-= GETPC_ADJ
;
318 if (searched_pc
< host_pc
) {
322 /* Reconstruct the stored insn data while looking for the point at
323 which the end of the insn exceeds the searched_pc. */
324 for (i
= 0; i
< num_insns
; ++i
) {
325 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
326 data
[j
] += decode_sleb128(&p
);
328 host_pc
+= decode_sleb128(&p
);
329 if (host_pc
> searched_pc
) {
336 if (tb
->cflags
& CF_USE_ICOUNT
) {
338 /* Reset the cycle counter to the start of the block. */
339 cpu
->icount_decr
.u16
.low
+= num_insns
;
340 /* Clear the IO flag. */
343 cpu
->icount_decr
.u16
.low
-= i
;
344 restore_state_to_opc(env
, tb
, data
);
346 #ifdef CONFIG_PROFILER
347 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
348 tcg_ctx
.restore_count
++;
353 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
355 TranslationBlock
*tb
;
358 /* A retaddr of zero is invalid so we really shouldn't have ended
359 * up here. The target code has likely forgotten to check retaddr
360 * != 0 before attempting to restore state. We return early to
361 * avoid blowing up on a recursive tb_lock(). The target must have
362 * previously survived a failed cpu_restore_state because
363 * tb_find_pc(0) would have failed anyway. It still should be
372 tb
= tb_find_pc(retaddr
);
374 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
375 if (tb
->cflags
& CF_NOCACHE
) {
376 /* one-shot translation, invalidate it immediately */
377 tb_phys_invalidate(tb
, -1);
387 static void page_init(void)
390 page_table_config_init();
392 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
394 #ifdef HAVE_KINFO_GETVMMAP
395 struct kinfo_vmentry
*freep
;
398 freep
= kinfo_getvmmap(getpid(), &cnt
);
401 for (i
= 0; i
< cnt
; i
++) {
402 unsigned long startaddr
, endaddr
;
404 startaddr
= freep
[i
].kve_start
;
405 endaddr
= freep
[i
].kve_end
;
406 if (h2g_valid(startaddr
)) {
407 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
409 if (h2g_valid(endaddr
)) {
410 endaddr
= h2g(endaddr
);
411 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
413 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
415 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
426 last_brk
= (unsigned long)sbrk(0);
428 f
= fopen("/compat/linux/proc/self/maps", "r");
433 unsigned long startaddr
, endaddr
;
436 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
438 if (n
== 2 && h2g_valid(startaddr
)) {
439 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
441 if (h2g_valid(endaddr
)) {
442 endaddr
= h2g(endaddr
);
446 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
459 * Called with tb_lock held for system emulation.
460 * Called with mmap_lock held for user-mode emulation.
462 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
469 assert_memory_lock();
472 /* Level 1. Always allocated. */
473 lp
= l1_map
+ ((index
>> v_l1_shift
) & (v_l1_size
- 1));
476 for (i
= v_l2_levels
; i
> 0; i
--) {
477 void **p
= atomic_rcu_read(lp
);
483 p
= g_new0(void *, V_L2_SIZE
);
484 atomic_rcu_set(lp
, p
);
487 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
490 pd
= atomic_rcu_read(lp
);
495 pd
= g_new0(PageDesc
, V_L2_SIZE
);
496 atomic_rcu_set(lp
, pd
);
499 return pd
+ (index
& (V_L2_SIZE
- 1));
502 static inline PageDesc
*page_find(tb_page_addr_t index
)
504 return page_find_alloc(index
, 0);
507 #if defined(CONFIG_USER_ONLY)
508 /* Currently it is not recommended to allocate big chunks of data in
509 user mode. It will change when a dedicated libc will be used. */
510 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
511 region in which the guest needs to run. Revisit this. */
512 #define USE_STATIC_CODE_GEN_BUFFER
515 /* Minimum size of the code gen buffer. This number is randomly chosen,
516 but not so small that we can't have a fair number of TB's live. */
517 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
519 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
520 indicated, this is constrained by the range of direct branches on the
521 host cpu, as used by the TCG implementation of goto_tb. */
522 #if defined(__x86_64__)
523 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
524 #elif defined(__sparc__)
525 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
526 #elif defined(__powerpc64__)
527 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
528 #elif defined(__powerpc__)
529 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
530 #elif defined(__aarch64__)
531 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
532 #elif defined(__s390x__)
533 /* We have a +- 4GB range on the branches; leave some slop. */
534 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
535 #elif defined(__mips__)
536 /* We have a 256MB branch region, but leave room to make sure the
537 main executable is also within that region. */
538 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
540 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
543 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
545 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
546 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
547 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
549 static inline size_t size_code_gen_buffer(size_t tb_size
)
551 /* Size the buffer. */
553 #ifdef USE_STATIC_CODE_GEN_BUFFER
554 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
556 /* ??? Needs adjustments. */
557 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
558 static buffer, we could size this on RESERVED_VA, on the text
559 segment size of the executable, or continue to use the default. */
560 tb_size
= (unsigned long)(ram_size
/ 4);
563 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
564 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
566 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
567 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
573 /* In order to use J and JAL within the code_gen_buffer, we require
574 that the buffer not cross a 256MB boundary. */
575 static inline bool cross_256mb(void *addr
, size_t size
)
577 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
580 /* We weren't able to allocate a buffer without crossing that boundary,
581 so make do with the larger portion of the buffer that doesn't cross.
582 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
583 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
585 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
586 size_t size2
= buf1
+ size1
- buf2
;
594 tcg_ctx
.code_gen_buffer_size
= size1
;
599 #ifdef USE_STATIC_CODE_GEN_BUFFER
600 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
601 __attribute__((aligned(CODE_GEN_ALIGN
)));
604 static inline void do_protect(void *addr
, long size
, int prot
)
607 VirtualProtect(addr
, size
, prot
, &old_protect
);
610 static inline void map_exec(void *addr
, long size
)
612 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
615 static inline void map_none(void *addr
, long size
)
617 do_protect(addr
, size
, PAGE_NOACCESS
);
620 static inline void do_protect(void *addr
, long size
, int prot
)
622 uintptr_t start
, end
;
624 start
= (uintptr_t)addr
;
625 start
&= qemu_real_host_page_mask
;
627 end
= (uintptr_t)addr
+ size
;
628 end
= ROUND_UP(end
, qemu_real_host_page_size
);
630 mprotect((void *)start
, end
- start
, prot
);
633 static inline void map_exec(void *addr
, long size
)
635 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
638 static inline void map_none(void *addr
, long size
)
640 do_protect(addr
, size
, PROT_NONE
);
644 static inline void *alloc_code_gen_buffer(void)
646 void *buf
= static_code_gen_buffer
;
647 size_t full_size
, size
;
649 /* The size of the buffer, rounded down to end on a page boundary. */
650 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
651 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
653 /* Reserve a guard page. */
654 size
= full_size
- qemu_real_host_page_size
;
656 /* Honor a command-line option limiting the size of the buffer. */
657 if (size
> tcg_ctx
.code_gen_buffer_size
) {
658 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
659 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
661 tcg_ctx
.code_gen_buffer_size
= size
;
664 if (cross_256mb(buf
, size
)) {
665 buf
= split_cross_256mb(buf
, size
);
666 size
= tcg_ctx
.code_gen_buffer_size
;
671 map_none(buf
+ size
, qemu_real_host_page_size
);
672 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
676 #elif defined(_WIN32)
677 static inline void *alloc_code_gen_buffer(void)
679 size_t size
= tcg_ctx
.code_gen_buffer_size
;
682 /* Perform the allocation in two steps, so that the guard page
683 is reserved but uncommitted. */
684 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
685 MEM_RESERVE
, PAGE_NOACCESS
);
687 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
688 assert(buf1
== buf2
);
694 static inline void *alloc_code_gen_buffer(void)
696 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
698 size_t size
= tcg_ctx
.code_gen_buffer_size
;
701 /* Constrain the position of the buffer based on the host cpu.
702 Note that these addresses are chosen in concert with the
703 addresses assigned in the relevant linker script file. */
704 # if defined(__PIE__) || defined(__PIC__)
705 /* Don't bother setting a preferred location if we're building
706 a position-independent executable. We're more likely to get
707 an address near the main executable if we let the kernel
708 choose the address. */
709 # elif defined(__x86_64__) && defined(MAP_32BIT)
710 /* Force the memory down into low memory with the executable.
711 Leave the choice of exact location with the kernel. */
713 /* Cannot expect to map more than 800MB in low memory. */
714 if (size
> 800u * 1024 * 1024) {
715 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
717 # elif defined(__sparc__)
718 start
= 0x40000000ul
;
719 # elif defined(__s390x__)
720 start
= 0x90000000ul
;
721 # elif defined(__mips__)
722 # if _MIPS_SIM == _ABI64
723 start
= 0x128000000ul
;
725 start
= 0x08000000ul
;
729 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
730 PROT_NONE
, flags
, -1, 0);
731 if (buf
== MAP_FAILED
) {
736 if (cross_256mb(buf
, size
)) {
737 /* Try again, with the original still mapped, to avoid re-acquiring
738 that 256mb crossing. This time don't specify an address. */
740 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
741 PROT_NONE
, flags
, -1, 0);
742 switch ((int)(buf2
!= MAP_FAILED
)) {
744 if (!cross_256mb(buf2
, size
)) {
745 /* Success! Use the new buffer. */
746 munmap(buf
, size
+ qemu_real_host_page_size
);
749 /* Failure. Work with what we had. */
750 munmap(buf2
, size
+ qemu_real_host_page_size
);
753 /* Split the original buffer. Free the smaller half. */
754 buf2
= split_cross_256mb(buf
, size
);
755 size2
= tcg_ctx
.code_gen_buffer_size
;
757 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
759 munmap(buf
, size
- size2
);
768 /* Make the final buffer accessible. The guard page at the end
769 will remain inaccessible with PROT_NONE. */
770 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
772 /* Request large pages for the buffer. */
773 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
777 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
779 static inline void code_gen_alloc(size_t tb_size
)
781 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
782 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
783 if (tcg_ctx
.code_gen_buffer
== NULL
) {
784 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
788 /* size this conservatively -- realloc later if needed */
789 tcg_ctx
.tb_ctx
.tbs_size
=
790 tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
/ 8;
791 if (unlikely(!tcg_ctx
.tb_ctx
.tbs_size
)) {
792 tcg_ctx
.tb_ctx
.tbs_size
= 64 * 1024;
794 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
*, tcg_ctx
.tb_ctx
.tbs_size
);
796 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
799 static void tb_htable_init(void)
801 unsigned int mode
= QHT_MODE_AUTO_RESIZE
;
803 qht_init(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
, mode
);
806 /* Must be called before using the QEMU cpus. 'tb_size' is the size
807 (in bytes) allocated to the translation buffer. Zero means default
809 void tcg_exec_init(unsigned long tb_size
)
815 code_gen_alloc(tb_size
);
816 #if defined(CONFIG_SOFTMMU)
817 /* There's no guest base to take into account, so go ahead and
818 initialize the prologue now. */
819 tcg_prologue_init(&tcg_ctx
);
824 * Allocate a new translation block. Flush the translation buffer if
825 * too many translation blocks or too much generated code.
827 * Called with tb_lock held.
829 static TranslationBlock
*tb_alloc(target_ulong pc
)
831 TranslationBlock
*tb
;
836 tb
= tcg_tb_alloc(&tcg_ctx
);
837 if (unlikely(tb
== NULL
)) {
840 ctx
= &tcg_ctx
.tb_ctx
;
841 if (unlikely(ctx
->nb_tbs
== ctx
->tbs_size
)) {
843 ctx
->tbs
= g_renew(TranslationBlock
*, ctx
->tbs
, ctx
->tbs_size
);
845 ctx
->tbs
[ctx
->nb_tbs
++] = tb
;
849 /* Called with tb_lock held. */
850 void tb_free(TranslationBlock
*tb
)
854 /* In practice this is mostly used for single use temporary TB
855 Ignore the hard cases and just back up if this TB happens to
856 be the last one generated. */
857 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
858 tb
== tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
859 size_t struct_size
= ROUND_UP(sizeof(*tb
), qemu_icache_linesize
);
861 tcg_ctx
.code_gen_ptr
= tb
->tc
.ptr
- struct_size
;
862 tcg_ctx
.tb_ctx
.nb_tbs
--;
866 static inline void invalidate_page_bitmap(PageDesc
*p
)
868 #ifdef CONFIG_SOFTMMU
869 g_free(p
->code_bitmap
);
870 p
->code_bitmap
= NULL
;
871 p
->code_write_count
= 0;
875 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
876 static void page_flush_tb_1(int level
, void **lp
)
886 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
887 pd
[i
].first_tb
= NULL
;
888 invalidate_page_bitmap(pd
+ i
);
893 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
894 page_flush_tb_1(level
- 1, pp
+ i
);
899 static void page_flush_tb(void)
901 int i
, l1_sz
= v_l1_size
;
903 for (i
= 0; i
< l1_sz
; i
++) {
904 page_flush_tb_1(v_l2_levels
, l1_map
+ i
);
908 /* flush all the translation blocks */
909 static void do_tb_flush(CPUState
*cpu
, run_on_cpu_data tb_flush_count
)
913 /* If it is already been done on request of another CPU,
916 if (tcg_ctx
.tb_ctx
.tb_flush_count
!= tb_flush_count
.host_int
) {
920 if (DEBUG_TB_FLUSH_GATE
) {
921 printf("qemu: flush code_size=%td nb_tbs=%d avg_tb_size=%td\n",
922 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
923 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
924 (tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
) /
925 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
927 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
928 > tcg_ctx
.code_gen_buffer_size
) {
929 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
933 cpu_tb_jmp_cache_clear(cpu
);
936 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
937 qht_reset_size(&tcg_ctx
.tb_ctx
.htable
, CODE_GEN_HTABLE_SIZE
);
940 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
941 /* XXX: flush processor icache at this point if cache flush is
943 atomic_mb_set(&tcg_ctx
.tb_ctx
.tb_flush_count
,
944 tcg_ctx
.tb_ctx
.tb_flush_count
+ 1);
950 void tb_flush(CPUState
*cpu
)
953 unsigned tb_flush_count
= atomic_mb_read(&tcg_ctx
.tb_ctx
.tb_flush_count
);
954 async_safe_run_on_cpu(cpu
, do_tb_flush
,
955 RUN_ON_CPU_HOST_INT(tb_flush_count
));
960 * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
961 * so in order to prevent bit rot we compile them unconditionally in user-mode,
962 * and let the optimizer get rid of them by wrapping their user-only callers
963 * with if (DEBUG_TB_CHECK_GATE).
965 #ifdef CONFIG_USER_ONLY
968 do_tb_invalidate_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
970 TranslationBlock
*tb
= p
;
971 target_ulong addr
= *(target_ulong
*)userp
;
973 if (!(addr
+ TARGET_PAGE_SIZE
<= tb
->pc
|| addr
>= tb
->pc
+ tb
->size
)) {
974 printf("ERROR invalidate: address=" TARGET_FMT_lx
975 " PC=%08lx size=%04x\n", addr
, (long)tb
->pc
, tb
->size
);
979 /* verify that all the pages have correct rights for code
981 * Called with tb_lock held.
983 static void tb_invalidate_check(target_ulong address
)
985 address
&= TARGET_PAGE_MASK
;
986 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_invalidate_check
, &address
);
990 do_tb_page_check(struct qht
*ht
, void *p
, uint32_t hash
, void *userp
)
992 TranslationBlock
*tb
= p
;
995 flags1
= page_get_flags(tb
->pc
);
996 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
997 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
998 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
999 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
1003 /* verify that all the pages have correct rights for code */
1004 static void tb_page_check(void)
1006 qht_iter(&tcg_ctx
.tb_ctx
.htable
, do_tb_page_check
, NULL
);
1009 #endif /* CONFIG_USER_ONLY */
1011 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
1013 TranslationBlock
*tb1
;
1018 n1
= (uintptr_t)tb1
& 3;
1019 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
1021 *ptb
= tb1
->page_next
[n1
];
1024 ptb
= &tb1
->page_next
[n1
];
1028 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
1029 static inline void tb_remove_from_jmp_list(TranslationBlock
*tb
, int n
)
1031 TranslationBlock
*tb1
;
1032 uintptr_t *ptb
, ntb
;
1035 ptb
= &tb
->jmp_list_next
[n
];
1037 /* find tb(n) in circular list */
1041 tb1
= (TranslationBlock
*)(ntb
& ~3);
1042 if (n1
== n
&& tb1
== tb
) {
1046 ptb
= &tb1
->jmp_list_first
;
1048 ptb
= &tb1
->jmp_list_next
[n1
];
1051 /* now we can suppress tb(n) from the list */
1052 *ptb
= tb
->jmp_list_next
[n
];
1054 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
1058 /* reset the jump entry 'n' of a TB so that it is not chained to
1060 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
1062 uintptr_t addr
= (uintptr_t)(tb
->tc
.ptr
+ tb
->jmp_reset_offset
[n
]);
1063 tb_set_jmp_target(tb
, n
, addr
);
1066 /* remove any jumps to the TB */
1067 static inline void tb_jmp_unlink(TranslationBlock
*tb
)
1069 TranslationBlock
*tb1
;
1070 uintptr_t *ptb
, ntb
;
1073 ptb
= &tb
->jmp_list_first
;
1077 tb1
= (TranslationBlock
*)(ntb
& ~3);
1081 tb_reset_jump(tb1
, n1
);
1082 *ptb
= tb1
->jmp_list_next
[n1
];
1083 tb1
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
1087 /* invalidate one TB
1089 * Called with tb_lock held.
1091 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
1096 tb_page_addr_t phys_pc
;
1100 atomic_set(&tb
->cflags
, tb
->cflags
| CF_INVALID
);
1102 /* remove the TB from the hash list */
1103 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1104 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->cflags
& CF_HASH_MASK
,
1105 tb
->trace_vcpu_dstate
);
1106 qht_remove(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1108 /* remove the TB from the page list */
1109 if (tb
->page_addr
[0] != page_addr
) {
1110 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
1111 tb_page_remove(&p
->first_tb
, tb
);
1112 invalidate_page_bitmap(p
);
1114 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
1115 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
1116 tb_page_remove(&p
->first_tb
, tb
);
1117 invalidate_page_bitmap(p
);
1120 /* remove the TB from the hash list */
1121 h
= tb_jmp_cache_hash_func(tb
->pc
);
1123 if (atomic_read(&cpu
->tb_jmp_cache
[h
]) == tb
) {
1124 atomic_set(&cpu
->tb_jmp_cache
[h
], NULL
);
1128 /* suppress this TB from the two jump lists */
1129 tb_remove_from_jmp_list(tb
, 0);
1130 tb_remove_from_jmp_list(tb
, 1);
1132 /* suppress any remaining jumps to this TB */
1135 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1138 #ifdef CONFIG_SOFTMMU
1139 static void build_page_bitmap(PageDesc
*p
)
1141 int n
, tb_start
, tb_end
;
1142 TranslationBlock
*tb
;
1144 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1147 while (tb
!= NULL
) {
1148 n
= (uintptr_t)tb
& 3;
1149 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1150 /* NOTE: this is subtle as a TB may span two physical pages */
1152 /* NOTE: tb_end may be after the end of the page, but
1153 it is not a problem */
1154 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1155 tb_end
= tb_start
+ tb
->size
;
1156 if (tb_end
> TARGET_PAGE_SIZE
) {
1157 tb_end
= TARGET_PAGE_SIZE
;
1161 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1163 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1164 tb
= tb
->page_next
[n
];
1169 /* add the tb in the target page and protect it if necessary
1171 * Called with mmap_lock held for user-mode emulation.
1173 static inline void tb_alloc_page(TranslationBlock
*tb
,
1174 unsigned int n
, tb_page_addr_t page_addr
)
1177 #ifndef CONFIG_USER_ONLY
1178 bool page_already_protected
;
1181 assert_memory_lock();
1183 tb
->page_addr
[n
] = page_addr
;
1184 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1185 tb
->page_next
[n
] = p
->first_tb
;
1186 #ifndef CONFIG_USER_ONLY
1187 page_already_protected
= p
->first_tb
!= NULL
;
1189 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1190 invalidate_page_bitmap(p
);
1192 #if defined(CONFIG_USER_ONLY)
1193 if (p
->flags
& PAGE_WRITE
) {
1198 /* force the host page as non writable (writes will have a
1199 page fault + mprotect overhead) */
1200 page_addr
&= qemu_host_page_mask
;
1202 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1203 addr
+= TARGET_PAGE_SIZE
) {
1205 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1210 p2
->flags
&= ~PAGE_WRITE
;
1212 mprotect(g2h(page_addr
), qemu_host_page_size
,
1213 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1214 if (DEBUG_TB_INVALIDATE_GATE
) {
1215 printf("protecting code page: 0x" TB_PAGE_ADDR_FMT
"\n", page_addr
);
1219 /* if some code is already present, then the pages are already
1220 protected. So we handle the case where only the first TB is
1221 allocated in a physical page */
1222 if (!page_already_protected
) {
1223 tlb_protect_code(page_addr
);
1228 /* add a new TB and link it to the physical page tables. phys_page2 is
1229 * (-1) to indicate that only one page contains the TB.
1231 * Called with mmap_lock held for user-mode emulation.
1233 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1234 tb_page_addr_t phys_page2
)
1238 assert_memory_lock();
1240 /* add in the page list */
1241 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1242 if (phys_page2
!= -1) {
1243 tb_alloc_page(tb
, 1, phys_page2
);
1245 tb
->page_addr
[1] = -1;
1248 /* add in the hash table */
1249 h
= tb_hash_func(phys_pc
, tb
->pc
, tb
->flags
, tb
->cflags
& CF_HASH_MASK
,
1250 tb
->trace_vcpu_dstate
);
1251 qht_insert(&tcg_ctx
.tb_ctx
.htable
, tb
, h
);
1253 #ifdef CONFIG_USER_ONLY
1254 if (DEBUG_TB_CHECK_GATE
) {
1260 /* Called with mmap_lock held for user mode emulation. */
1261 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1262 target_ulong pc
, target_ulong cs_base
,
1263 uint32_t flags
, int cflags
)
1265 CPUArchState
*env
= cpu
->env_ptr
;
1266 TranslationBlock
*tb
;
1267 tb_page_addr_t phys_pc
, phys_page2
;
1268 target_ulong virt_page2
;
1269 tcg_insn_unit
*gen_code_buf
;
1270 int gen_code_size
, search_size
;
1271 #ifdef CONFIG_PROFILER
1274 assert_memory_lock();
1276 phys_pc
= get_page_addr_code(env
, pc
);
1279 if (unlikely(!tb
)) {
1281 /* flush must be done */
1284 /* Make the execution loop process the flush as soon as possible. */
1285 cpu
->exception_index
= EXCP_INTERRUPT
;
1289 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1290 tb
->tc
.ptr
= gen_code_buf
;
1292 tb
->cs_base
= cs_base
;
1294 tb
->cflags
= cflags
;
1295 tb
->trace_vcpu_dstate
= *cpu
->trace_dstate
;
1296 tcg_ctx
.tb_cflags
= cflags
;
1298 #ifdef CONFIG_PROFILER
1299 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1301 ti
= profile_getclock();
1304 tcg_func_start(&tcg_ctx
);
1306 tcg_ctx
.cpu
= ENV_GET_CPU(env
);
1307 gen_intermediate_code(cpu
, tb
);
1310 trace_translate_block(tb
, tb
->pc
, tb
->tc
.ptr
);
1312 /* generate machine code */
1313 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1314 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1315 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1316 if (TCG_TARGET_HAS_direct_jump
) {
1317 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_target_arg
;
1318 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1320 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1321 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_arg
;
1324 #ifdef CONFIG_PROFILER
1326 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1327 ti
= profile_getclock();
1330 /* ??? Overflow could be handled better here. In particular, we
1331 don't need to re-do gen_intermediate_code, nor should we re-do
1332 the tcg optimization currently hidden inside tcg_gen_code. All
1333 that should be required is to flush the TBs, allocate a new TB,
1334 re-initialize it per above, and re-do the actual code generation. */
1335 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1336 if (unlikely(gen_code_size
< 0)) {
1337 goto buffer_overflow
;
1339 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1340 if (unlikely(search_size
< 0)) {
1341 goto buffer_overflow
;
1344 #ifdef CONFIG_PROFILER
1345 tcg_ctx
.code_time
+= profile_getclock() - ti
;
1346 tcg_ctx
.code_in_len
+= tb
->size
;
1347 tcg_ctx
.code_out_len
+= gen_code_size
;
1348 tcg_ctx
.search_out_len
+= search_size
;
1352 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1353 qemu_log_in_addr_range(tb
->pc
)) {
1355 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1356 if (tcg_ctx
.data_gen_ptr
) {
1357 size_t code_size
= tcg_ctx
.data_gen_ptr
- tb
->tc
.ptr
;
1358 size_t data_size
= gen_code_size
- code_size
;
1361 log_disas(tb
->tc
.ptr
, code_size
);
1363 for (i
= 0; i
< data_size
; i
+= sizeof(tcg_target_ulong
)) {
1364 if (sizeof(tcg_target_ulong
) == 8) {
1365 qemu_log("0x%08" PRIxPTR
": .quad 0x%016" PRIx64
"\n",
1366 (uintptr_t)tcg_ctx
.data_gen_ptr
+ i
,
1367 *(uint64_t *)(tcg_ctx
.data_gen_ptr
+ i
));
1369 qemu_log("0x%08" PRIxPTR
": .long 0x%08x\n",
1370 (uintptr_t)tcg_ctx
.data_gen_ptr
+ i
,
1371 *(uint32_t *)(tcg_ctx
.data_gen_ptr
+ i
));
1375 log_disas(tb
->tc
.ptr
, gen_code_size
);
1383 tcg_ctx
.code_gen_ptr
= (void *)
1384 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1387 /* init jump list */
1388 assert(((uintptr_t)tb
& 3) == 0);
1389 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1390 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1391 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1393 /* init original jump addresses wich has been set during tcg_gen_code() */
1394 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1395 tb_reset_jump(tb
, 0);
1397 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1398 tb_reset_jump(tb
, 1);
1401 /* check next page if needed */
1402 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1404 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1405 phys_page2
= get_page_addr_code(env
, virt_page2
);
1407 /* As long as consistency of the TB stuff is provided by tb_lock in user
1408 * mode and is implicit in single-threaded softmmu emulation, no explicit
1409 * memory barrier is required before tb_link_page() makes the TB visible
1410 * through the physical hash table and physical page list.
1412 tb_link_page(tb
, phys_pc
, phys_page2
);
1417 * Invalidate all TBs which intersect with the target physical address range
1418 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1419 * 'is_cpu_write_access' should be true if called from a real cpu write
1420 * access: the virtual CPU will exit the current TB if code is modified inside
1423 * Called with mmap_lock held for user-mode emulation, grabs tb_lock
1424 * Called with tb_lock held for system-mode emulation
1426 static void tb_invalidate_phys_range_1(tb_page_addr_t start
, tb_page_addr_t end
)
1428 while (start
< end
) {
1429 tb_invalidate_phys_page_range(start
, end
, 0);
1430 start
&= TARGET_PAGE_MASK
;
1431 start
+= TARGET_PAGE_SIZE
;
1435 #ifdef CONFIG_SOFTMMU
1436 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1439 tb_invalidate_phys_range_1(start
, end
);
1442 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1444 assert_memory_lock();
1446 tb_invalidate_phys_range_1(start
, end
);
1451 * Invalidate all TBs which intersect with the target physical address range
1452 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1453 * 'is_cpu_write_access' should be true if called from a real cpu write
1454 * access: the virtual CPU will exit the current TB if code is modified inside
1457 * Called with tb_lock/mmap_lock held for user-mode emulation
1458 * Called with tb_lock held for system-mode emulation
1460 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1461 int is_cpu_write_access
)
1463 TranslationBlock
*tb
, *tb_next
;
1464 tb_page_addr_t tb_start
, tb_end
;
1467 #ifdef TARGET_HAS_PRECISE_SMC
1468 CPUState
*cpu
= current_cpu
;
1469 CPUArchState
*env
= NULL
;
1470 int current_tb_not_found
= is_cpu_write_access
;
1471 TranslationBlock
*current_tb
= NULL
;
1472 int current_tb_modified
= 0;
1473 target_ulong current_pc
= 0;
1474 target_ulong current_cs_base
= 0;
1475 uint32_t current_flags
= 0;
1476 #endif /* TARGET_HAS_PRECISE_SMC */
1478 assert_memory_lock();
1481 p
= page_find(start
>> TARGET_PAGE_BITS
);
1485 #if defined(TARGET_HAS_PRECISE_SMC)
1491 /* we remove all the TBs in the range [start, end[ */
1492 /* XXX: see if in some cases it could be faster to invalidate all
1495 while (tb
!= NULL
) {
1496 n
= (uintptr_t)tb
& 3;
1497 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1498 tb_next
= tb
->page_next
[n
];
1499 /* NOTE: this is subtle as a TB may span two physical pages */
1501 /* NOTE: tb_end may be after the end of the page, but
1502 it is not a problem */
1503 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1504 tb_end
= tb_start
+ tb
->size
;
1506 tb_start
= tb
->page_addr
[1];
1507 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1509 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1510 #ifdef TARGET_HAS_PRECISE_SMC
1511 if (current_tb_not_found
) {
1512 current_tb_not_found
= 0;
1514 if (cpu
->mem_io_pc
) {
1515 /* now we have a real cpu fault */
1516 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1519 if (current_tb
== tb
&&
1520 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1521 /* If we are modifying the current TB, we must stop
1522 its execution. We could be more precise by checking
1523 that the modification is after the current PC, but it
1524 would require a specialized function to partially
1525 restore the CPU state */
1527 current_tb_modified
= 1;
1528 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1529 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1532 #endif /* TARGET_HAS_PRECISE_SMC */
1533 tb_phys_invalidate(tb
, -1);
1537 #if !defined(CONFIG_USER_ONLY)
1538 /* if no code remaining, no need to continue to use slow writes */
1540 invalidate_page_bitmap(p
);
1541 tlb_unprotect_code(start
);
1544 #ifdef TARGET_HAS_PRECISE_SMC
1545 if (current_tb_modified
) {
1546 /* Force execution of one insn next time. */
1547 cpu
->cflags_next_tb
= 1 | curr_cflags();
1548 cpu_loop_exit_noexc(cpu
);
1553 #ifdef CONFIG_SOFTMMU
1554 /* len must be <= 8 and start must be a multiple of len.
1555 * Called via softmmu_template.h when code areas are written to with
1556 * iothread mutex not held.
1558 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1564 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1565 cpu_single_env
->mem_io_vaddr
, len
,
1566 cpu_single_env
->eip
,
1567 cpu_single_env
->eip
+
1568 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1571 assert_memory_lock();
1573 p
= page_find(start
>> TARGET_PAGE_BITS
);
1577 if (!p
->code_bitmap
&&
1578 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1579 /* build code bitmap. FIXME: writes should be protected by
1580 * tb_lock, reads by tb_lock or RCU.
1582 build_page_bitmap(p
);
1584 if (p
->code_bitmap
) {
1588 nr
= start
& ~TARGET_PAGE_MASK
;
1589 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1590 if (b
& ((1 << len
) - 1)) {
1595 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1599 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1600 * host PC of the faulting store instruction that caused this invalidate.
1601 * Returns true if the caller needs to abort execution of the current
1602 * TB (because it was modified by this store and the guest CPU has
1603 * precise-SMC semantics).
1605 static bool tb_invalidate_phys_page(tb_page_addr_t addr
, uintptr_t pc
)
1607 TranslationBlock
*tb
;
1610 #ifdef TARGET_HAS_PRECISE_SMC
1611 TranslationBlock
*current_tb
= NULL
;
1612 CPUState
*cpu
= current_cpu
;
1613 CPUArchState
*env
= NULL
;
1614 int current_tb_modified
= 0;
1615 target_ulong current_pc
= 0;
1616 target_ulong current_cs_base
= 0;
1617 uint32_t current_flags
= 0;
1620 assert_memory_lock();
1622 addr
&= TARGET_PAGE_MASK
;
1623 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1630 #ifdef TARGET_HAS_PRECISE_SMC
1631 if (tb
&& pc
!= 0) {
1632 current_tb
= tb_find_pc(pc
);
1638 while (tb
!= NULL
) {
1639 n
= (uintptr_t)tb
& 3;
1640 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1641 #ifdef TARGET_HAS_PRECISE_SMC
1642 if (current_tb
== tb
&&
1643 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1644 /* If we are modifying the current TB, we must stop
1645 its execution. We could be more precise by checking
1646 that the modification is after the current PC, but it
1647 would require a specialized function to partially
1648 restore the CPU state */
1650 current_tb_modified
= 1;
1651 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1652 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1655 #endif /* TARGET_HAS_PRECISE_SMC */
1656 tb_phys_invalidate(tb
, addr
);
1657 tb
= tb
->page_next
[n
];
1660 #ifdef TARGET_HAS_PRECISE_SMC
1661 if (current_tb_modified
) {
1662 /* Force execution of one insn next time. */
1663 cpu
->cflags_next_tb
= 1 | curr_cflags();
1664 /* tb_lock will be reset after cpu_loop_exit_noexc longjmps
1665 * back into the cpu_exec loop. */
1675 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1676 tb[1].tc_ptr. Return NULL if not found */
1677 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1679 int m_min
, m_max
, m
;
1681 TranslationBlock
*tb
;
1683 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1686 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1687 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1690 /* binary search (cf Knuth) */
1692 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1693 while (m_min
<= m_max
) {
1694 m
= (m_min
+ m_max
) >> 1;
1695 tb
= tcg_ctx
.tb_ctx
.tbs
[m
];
1696 v
= (uintptr_t)tb
->tc
.ptr
;
1699 } else if (tc_ptr
< v
) {
1705 return tcg_ctx
.tb_ctx
.tbs
[m_max
];
1708 #if !defined(CONFIG_USER_ONLY)
1709 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1711 ram_addr_t ram_addr
;
1716 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1717 if (!(memory_region_is_ram(mr
)
1718 || memory_region_is_romd(mr
))) {
1722 ram_addr
= memory_region_get_ram_addr(mr
) + addr
;
1724 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1728 #endif /* !defined(CONFIG_USER_ONLY) */
1730 /* Called with tb_lock held. */
1731 void tb_check_watchpoint(CPUState
*cpu
)
1733 TranslationBlock
*tb
;
1735 tb
= tb_find_pc(cpu
->mem_io_pc
);
1737 /* We can use retranslation to find the PC. */
1738 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1739 tb_phys_invalidate(tb
, -1);
1741 /* The exception probably happened in a helper. The CPU state should
1742 have been saved before calling it. Fetch the PC from there. */
1743 CPUArchState
*env
= cpu
->env_ptr
;
1744 target_ulong pc
, cs_base
;
1745 tb_page_addr_t addr
;
1748 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1749 addr
= get_page_addr_code(env
, pc
);
1750 tb_invalidate_phys_range(addr
, addr
+ 1);
1754 #ifndef CONFIG_USER_ONLY
1755 /* in deterministic execution mode, instructions doing device I/Os
1756 * must be at the end of the TB.
1758 * Called by softmmu_template.h, with iothread mutex not held.
1760 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1762 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1763 CPUArchState
*env
= cpu
->env_ptr
;
1765 TranslationBlock
*tb
;
1769 tb
= tb_find_pc(retaddr
);
1771 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1774 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1775 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1776 /* Calculate how many instructions had been executed before the fault
1778 n
= n
- cpu
->icount_decr
.u16
.low
;
1779 /* Generate a new TB ending on the I/O insn. */
1781 /* On MIPS and SH, delay slot instructions can only be restarted if
1782 they were already the first instruction in the TB. If this is not
1783 the first instruction in a TB then re-execute the preceding
1785 #if defined(TARGET_MIPS)
1786 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1787 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1788 cpu
->icount_decr
.u16
.low
++;
1789 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1791 #elif defined(TARGET_SH4)
1792 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1795 cpu
->icount_decr
.u16
.low
++;
1796 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1799 /* This should never happen. */
1800 if (n
> CF_COUNT_MASK
) {
1801 cpu_abort(cpu
, "TB too big during recompile");
1804 /* Adjust the execution state of the next TB. */
1805 cpu
->cflags_next_tb
= curr_cflags() | CF_LAST_IO
| n
;
1807 if (tb
->cflags
& CF_NOCACHE
) {
1809 /* Invalidate original TB if this TB was generated in
1810 * cpu_exec_nocache() */
1811 tb_phys_invalidate(tb
->orig_tb
, -1);
1816 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1817 * the first in the TB) then we end up generating a whole new TB and
1818 * repeating the fault, which is horribly inefficient.
1819 * Better would be to execute just this insn uncached, or generate a
1822 * cpu_loop_exit_noexc will longjmp back to cpu_exec where the
1823 * tb_lock gets reset.
1825 cpu_loop_exit_noexc(cpu
);
1828 static void tb_jmp_cache_clear_page(CPUState
*cpu
, target_ulong page_addr
)
1830 unsigned int i
, i0
= tb_jmp_cache_hash_page(page_addr
);
1832 for (i
= 0; i
< TB_JMP_PAGE_SIZE
; i
++) {
1833 atomic_set(&cpu
->tb_jmp_cache
[i0
+ i
], NULL
);
1837 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1839 /* Discard jump cache entries for any tb which might potentially
1840 overlap the flushed page. */
1841 tb_jmp_cache_clear_page(cpu
, addr
- TARGET_PAGE_SIZE
);
1842 tb_jmp_cache_clear_page(cpu
, addr
);
1845 static void print_qht_statistics(FILE *f
, fprintf_function cpu_fprintf
,
1846 struct qht_stats hst
)
1848 uint32_t hgram_opts
;
1852 if (!hst
.head_buckets
) {
1855 cpu_fprintf(f
, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1856 hst
.used_head_buckets
, hst
.head_buckets
,
1857 (double)hst
.used_head_buckets
/ hst
.head_buckets
* 100);
1859 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1860 hgram_opts
|= QDIST_PR_100X
| QDIST_PR_PERCENT
;
1861 if (qdist_xmax(&hst
.occupancy
) - qdist_xmin(&hst
.occupancy
) == 1) {
1862 hgram_opts
|= QDIST_PR_NODECIMAL
;
1864 hgram
= qdist_pr(&hst
.occupancy
, 10, hgram_opts
);
1865 cpu_fprintf(f
, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1866 qdist_avg(&hst
.occupancy
) * 100, hgram
);
1869 hgram_opts
= QDIST_PR_BORDER
| QDIST_PR_LABELS
;
1870 hgram_bins
= qdist_xmax(&hst
.chain
) - qdist_xmin(&hst
.chain
);
1871 if (hgram_bins
> 10) {
1875 hgram_opts
|= QDIST_PR_NODECIMAL
| QDIST_PR_NOBINRANGE
;
1877 hgram
= qdist_pr(&hst
.chain
, hgram_bins
, hgram_opts
);
1878 cpu_fprintf(f
, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1879 qdist_avg(&hst
.chain
), hgram
);
1883 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1885 int i
, target_code_size
, max_target_code_size
;
1886 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1887 TranslationBlock
*tb
;
1888 struct qht_stats hst
;
1892 target_code_size
= 0;
1893 max_target_code_size
= 0;
1895 direct_jmp_count
= 0;
1896 direct_jmp2_count
= 0;
1897 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1898 tb
= tcg_ctx
.tb_ctx
.tbs
[i
];
1899 target_code_size
+= tb
->size
;
1900 if (tb
->size
> max_target_code_size
) {
1901 max_target_code_size
= tb
->size
;
1903 if (tb
->page_addr
[1] != -1) {
1906 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1908 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1909 direct_jmp2_count
++;
1913 /* XXX: avoid using doubles ? */
1914 cpu_fprintf(f
, "Translation buffer state:\n");
1915 cpu_fprintf(f
, "gen code size %td/%zd\n",
1916 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1917 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1918 cpu_fprintf(f
, "TB count %d\n", tcg_ctx
.tb_ctx
.nb_tbs
);
1919 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1920 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1921 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1922 max_target_code_size
);
1923 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1924 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1925 tcg_ctx
.code_gen_buffer
) /
1926 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1927 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1928 tcg_ctx
.code_gen_buffer
) /
1929 target_code_size
: 0);
1930 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1931 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1932 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1933 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1935 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1936 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1938 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1939 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1941 qht_statistics_init(&tcg_ctx
.tb_ctx
.htable
, &hst
);
1942 print_qht_statistics(f
, cpu_fprintf
, hst
);
1943 qht_statistics_destroy(&hst
);
1945 cpu_fprintf(f
, "\nStatistics:\n");
1946 cpu_fprintf(f
, "TB flush count %u\n",
1947 atomic_read(&tcg_ctx
.tb_ctx
.tb_flush_count
));
1948 cpu_fprintf(f
, "TB invalidate count %d\n",
1949 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1950 cpu_fprintf(f
, "TLB flush count %zu\n", tlb_flush_count());
1951 tcg_dump_info(f
, cpu_fprintf
);
1956 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1958 tcg_dump_op_count(f
, cpu_fprintf
);
1961 #else /* CONFIG_USER_ONLY */
1963 void cpu_interrupt(CPUState
*cpu
, int mask
)
1965 g_assert(qemu_mutex_iothread_locked());
1966 cpu
->interrupt_request
|= mask
;
1967 cpu
->icount_decr
.u16
.high
= -1;
1971 * Walks guest process memory "regions" one by one
1972 * and calls callback function 'fn' for each region.
1974 struct walk_memory_regions_data
{
1975 walk_memory_regions_fn fn
;
1981 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1982 target_ulong end
, int new_prot
)
1984 if (data
->start
!= -1u) {
1985 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1991 data
->start
= (new_prot
? end
: -1u);
1992 data
->prot
= new_prot
;
1997 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1998 target_ulong base
, int level
, void **lp
)
2004 return walk_memory_regions_end(data
, base
, 0);
2010 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2011 int prot
= pd
[i
].flags
;
2013 pa
= base
| (i
<< TARGET_PAGE_BITS
);
2014 if (prot
!= data
->prot
) {
2015 rc
= walk_memory_regions_end(data
, pa
, prot
);
2024 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
2025 pa
= base
| ((target_ulong
)i
<<
2026 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
2027 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
2037 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
2039 struct walk_memory_regions_data data
;
2040 uintptr_t i
, l1_sz
= v_l1_size
;
2047 for (i
= 0; i
< l1_sz
; i
++) {
2048 target_ulong base
= i
<< (v_l1_shift
+ TARGET_PAGE_BITS
);
2049 int rc
= walk_memory_regions_1(&data
, base
, v_l2_levels
, l1_map
+ i
);
2055 return walk_memory_regions_end(&data
, 0, 0);
2058 static int dump_region(void *priv
, target_ulong start
,
2059 target_ulong end
, unsigned long prot
)
2061 FILE *f
= (FILE *)priv
;
2063 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
2064 " "TARGET_FMT_lx
" %c%c%c\n",
2065 start
, end
, end
- start
,
2066 ((prot
& PAGE_READ
) ? 'r' : '-'),
2067 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
2068 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
2073 /* dump memory mappings */
2074 void page_dump(FILE *f
)
2076 const int length
= sizeof(target_ulong
) * 2;
2077 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
2078 length
, "start", length
, "end", length
, "size", "prot");
2079 walk_memory_regions(f
, dump_region
);
2082 int page_get_flags(target_ulong address
)
2086 p
= page_find(address
>> TARGET_PAGE_BITS
);
2093 /* Modify the flags of a page and invalidate the code if necessary.
2094 The flag PAGE_WRITE_ORG is positioned automatically depending
2095 on PAGE_WRITE. The mmap_lock should already be held. */
2096 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
2098 target_ulong addr
, len
;
2100 /* This function should never be called with addresses outside the
2101 guest address space. If this assert fires, it probably indicates
2102 a missing call to h2g_valid. */
2103 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2104 assert(end
<= ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2106 assert(start
< end
);
2107 assert_memory_lock();
2109 start
= start
& TARGET_PAGE_MASK
;
2110 end
= TARGET_PAGE_ALIGN(end
);
2112 if (flags
& PAGE_WRITE
) {
2113 flags
|= PAGE_WRITE_ORG
;
2116 for (addr
= start
, len
= end
- start
;
2118 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2119 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
2121 /* If the write protection bit is set, then we invalidate
2123 if (!(p
->flags
& PAGE_WRITE
) &&
2124 (flags
& PAGE_WRITE
) &&
2126 tb_invalidate_phys_page(addr
, 0);
2132 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
2138 /* This function should never be called with addresses outside the
2139 guest address space. If this assert fires, it probably indicates
2140 a missing call to h2g_valid. */
2141 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
2142 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
2148 if (start
+ len
- 1 < start
) {
2149 /* We've wrapped around. */
2153 /* must do before we loose bits in the next step */
2154 end
= TARGET_PAGE_ALIGN(start
+ len
);
2155 start
= start
& TARGET_PAGE_MASK
;
2157 for (addr
= start
, len
= end
- start
;
2159 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
2160 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2164 if (!(p
->flags
& PAGE_VALID
)) {
2168 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
2171 if (flags
& PAGE_WRITE
) {
2172 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
2175 /* unprotect the page if it was put read-only because it
2176 contains translated code */
2177 if (!(p
->flags
& PAGE_WRITE
)) {
2178 if (!page_unprotect(addr
, 0)) {
2187 /* called from signal handler: invalidate the code and unprotect the
2188 * page. Return 0 if the fault was not handled, 1 if it was handled,
2189 * and 2 if it was handled but the caller must cause the TB to be
2190 * immediately exited. (We can only return 2 if the 'pc' argument is
2193 int page_unprotect(target_ulong address
, uintptr_t pc
)
2196 bool current_tb_invalidated
;
2198 target_ulong host_start
, host_end
, addr
;
2200 /* Technically this isn't safe inside a signal handler. However we
2201 know this only ever happens in a synchronous SEGV handler, so in
2202 practice it seems to be ok. */
2205 p
= page_find(address
>> TARGET_PAGE_BITS
);
2211 /* if the page was really writable, then we change its
2212 protection back to writable */
2213 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
2214 host_start
= address
& qemu_host_page_mask
;
2215 host_end
= host_start
+ qemu_host_page_size
;
2218 current_tb_invalidated
= false;
2219 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
2220 p
= page_find(addr
>> TARGET_PAGE_BITS
);
2221 p
->flags
|= PAGE_WRITE
;
2224 /* and since the content will be modified, we must invalidate
2225 the corresponding translated code. */
2226 current_tb_invalidated
|= tb_invalidate_phys_page(addr
, pc
);
2227 #ifdef CONFIG_USER_ONLY
2228 if (DEBUG_TB_CHECK_GATE
) {
2229 tb_invalidate_check(addr
);
2233 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2237 /* If current TB was invalidated return to main loop */
2238 return current_tb_invalidated
? 2 : 1;
2243 #endif /* CONFIG_USER_ONLY */
2245 /* This is a wrapper for common code that can not use CONFIG_SOFTMMU */
2246 void tcg_flush_softmmu_tlb(CPUState
*cs
)
2248 #ifdef CONFIG_SOFTMMU