4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
24 #include "qemu/osdep.h"
27 #include "qemu-common.h"
28 #define NO_CPU_IO_DEFS
31 #include "disas/disas.h"
33 #if defined(CONFIG_USER_ONLY)
35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
36 #include <sys/param.h>
37 #if __FreeBSD_version >= 700104
38 #define HAVE_KINFO_GETVMMAP
39 #define sigqueue sigqueue_freebsd /* avoid redefinition */
41 #include <machine/profile.h>
50 #include "exec/address-spaces.h"
53 #include "exec/cputlb.h"
54 #include "exec/tb-hash.h"
55 #include "translate-all.h"
56 #include "qemu/bitmap.h"
57 #include "qemu/timer.h"
60 //#define DEBUG_TB_INVALIDATE
62 /* make various TB consistency checks */
63 //#define DEBUG_TB_CHECK
65 #if !defined(CONFIG_USER_ONLY)
66 /* TB consistency checks only implemented for usermode emulation. */
70 #define SMC_BITMAP_USE_THRESHOLD 10
72 typedef struct PageDesc
{
73 /* list of TBs intersecting this ram page */
74 TranslationBlock
*first_tb
;
75 /* in order to optimize self modifying code, we count the number
76 of lookups we do to a given page to use a bitmap */
77 unsigned int code_write_count
;
78 unsigned long *code_bitmap
;
79 #if defined(CONFIG_USER_ONLY)
84 /* In system mode we want L1_MAP to be based on ram offsets,
85 while in user mode we want it to be based on virtual addresses. */
86 #if !defined(CONFIG_USER_ONLY)
87 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
88 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
90 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
96 /* Size of the L2 (and L3, etc) page tables. */
98 #define V_L2_SIZE (1 << V_L2_BITS)
100 /* The bits remaining after N lower levels of page tables. */
101 #define V_L1_BITS_REM \
102 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
104 #if V_L1_BITS_REM < 4
105 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
107 #define V_L1_BITS V_L1_BITS_REM
110 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
112 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
114 uintptr_t qemu_host_page_size
;
115 intptr_t qemu_host_page_mask
;
117 /* The bottom level has pointers to PageDesc */
118 static void *l1_map
[V_L1_SIZE
];
120 /* code generation context */
123 /* translation block context */
124 #ifdef CONFIG_USER_ONLY
125 __thread
int have_tb_lock
;
130 #ifdef CONFIG_USER_ONLY
131 assert(!have_tb_lock
);
132 qemu_mutex_lock(&tcg_ctx
.tb_ctx
.tb_lock
);
139 #ifdef CONFIG_USER_ONLY
140 assert(have_tb_lock
);
142 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
146 void tb_lock_reset(void)
148 #ifdef CONFIG_USER_ONLY
150 qemu_mutex_unlock(&tcg_ctx
.tb_ctx
.tb_lock
);
156 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
157 tb_page_addr_t phys_page2
);
158 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
);
160 void cpu_gen_init(void)
162 tcg_context_init(&tcg_ctx
);
165 /* Encode VAL as a signed leb128 sequence at P.
166 Return P incremented past the encoded value. */
167 static uint8_t *encode_sleb128(uint8_t *p
, target_long val
)
174 more
= !((val
== 0 && (byte
& 0x40) == 0)
175 || (val
== -1 && (byte
& 0x40) != 0));
185 /* Decode a signed leb128 sequence at *PP; increment *PP past the
186 decoded value. Return the decoded value. */
187 static target_long
decode_sleb128(uint8_t **pp
)
195 val
|= (target_ulong
)(byte
& 0x7f) << shift
;
197 } while (byte
& 0x80);
198 if (shift
< TARGET_LONG_BITS
&& (byte
& 0x40)) {
199 val
|= -(target_ulong
)1 << shift
;
206 /* Encode the data collected about the instructions while compiling TB.
207 Place the data at BLOCK, and return the number of bytes consumed.
209 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
210 which come from the target's insn_start data, followed by a uintptr_t
211 which comes from the host pc of the end of the code implementing the insn.
213 Each line of the table is encoded as sleb128 deltas from the previous
214 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
215 That is, the first column is seeded with the guest pc, the last column
216 with the host pc, and the middle columns with zeros. */
218 static int encode_search(TranslationBlock
*tb
, uint8_t *block
)
220 uint8_t *highwater
= tcg_ctx
.code_gen_highwater
;
224 tb
->tc_search
= block
;
226 for (i
= 0, n
= tb
->icount
; i
< n
; ++i
) {
229 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
231 prev
= (j
== 0 ? tb
->pc
: 0);
233 prev
= tcg_ctx
.gen_insn_data
[i
- 1][j
];
235 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_data
[i
][j
] - prev
);
237 prev
= (i
== 0 ? 0 : tcg_ctx
.gen_insn_end_off
[i
- 1]);
238 p
= encode_sleb128(p
, tcg_ctx
.gen_insn_end_off
[i
] - prev
);
240 /* Test for (pending) buffer overflow. The assumption is that any
241 one row beginning below the high water mark cannot overrun
242 the buffer completely. Thus we can test for overflow after
243 encoding a row without having to check during encoding. */
244 if (unlikely(p
> highwater
)) {
252 /* The cpu state corresponding to 'searched_pc' is restored. */
253 static int cpu_restore_state_from_tb(CPUState
*cpu
, TranslationBlock
*tb
,
254 uintptr_t searched_pc
)
256 target_ulong data
[TARGET_INSN_START_WORDS
] = { tb
->pc
};
257 uintptr_t host_pc
= (uintptr_t)tb
->tc_ptr
;
258 CPUArchState
*env
= cpu
->env_ptr
;
259 uint8_t *p
= tb
->tc_search
;
260 int i
, j
, num_insns
= tb
->icount
;
261 #ifdef CONFIG_PROFILER
262 int64_t ti
= profile_getclock();
265 if (searched_pc
< host_pc
) {
269 /* Reconstruct the stored insn data while looking for the point at
270 which the end of the insn exceeds the searched_pc. */
271 for (i
= 0; i
< num_insns
; ++i
) {
272 for (j
= 0; j
< TARGET_INSN_START_WORDS
; ++j
) {
273 data
[j
] += decode_sleb128(&p
);
275 host_pc
+= decode_sleb128(&p
);
276 if (host_pc
> searched_pc
) {
283 if (tb
->cflags
& CF_USE_ICOUNT
) {
285 /* Reset the cycle counter to the start of the block. */
286 cpu
->icount_decr
.u16
.low
+= num_insns
;
287 /* Clear the IO flag. */
290 cpu
->icount_decr
.u16
.low
-= i
;
291 restore_state_to_opc(env
, tb
, data
);
293 #ifdef CONFIG_PROFILER
294 tcg_ctx
.restore_time
+= profile_getclock() - ti
;
295 tcg_ctx
.restore_count
++;
300 bool cpu_restore_state(CPUState
*cpu
, uintptr_t retaddr
)
302 TranslationBlock
*tb
;
304 tb
= tb_find_pc(retaddr
);
306 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
307 if (tb
->cflags
& CF_NOCACHE
) {
308 /* one-shot translation, invalidate it immediately */
309 cpu
->current_tb
= NULL
;
310 tb_phys_invalidate(tb
, -1);
318 void page_size_init(void)
320 /* NOTE: we can always suppose that qemu_host_page_size >=
322 qemu_real_host_page_size
= getpagesize();
323 qemu_real_host_page_mask
= -(intptr_t)qemu_real_host_page_size
;
324 if (qemu_host_page_size
== 0) {
325 qemu_host_page_size
= qemu_real_host_page_size
;
327 if (qemu_host_page_size
< TARGET_PAGE_SIZE
) {
328 qemu_host_page_size
= TARGET_PAGE_SIZE
;
330 qemu_host_page_mask
= -(intptr_t)qemu_host_page_size
;
333 static void page_init(void)
336 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
338 #ifdef HAVE_KINFO_GETVMMAP
339 struct kinfo_vmentry
*freep
;
342 freep
= kinfo_getvmmap(getpid(), &cnt
);
345 for (i
= 0; i
< cnt
; i
++) {
346 unsigned long startaddr
, endaddr
;
348 startaddr
= freep
[i
].kve_start
;
349 endaddr
= freep
[i
].kve_end
;
350 if (h2g_valid(startaddr
)) {
351 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
353 if (h2g_valid(endaddr
)) {
354 endaddr
= h2g(endaddr
);
355 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
357 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
359 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
370 last_brk
= (unsigned long)sbrk(0);
372 f
= fopen("/compat/linux/proc/self/maps", "r");
377 unsigned long startaddr
, endaddr
;
380 n
= fscanf(f
, "%lx-%lx %*[^\n]\n", &startaddr
, &endaddr
);
382 if (n
== 2 && h2g_valid(startaddr
)) {
383 startaddr
= h2g(startaddr
) & TARGET_PAGE_MASK
;
385 if (h2g_valid(endaddr
)) {
386 endaddr
= h2g(endaddr
);
390 page_set_flags(startaddr
, endaddr
, PAGE_RESERVED
);
403 * Called with mmap_lock held for user-mode emulation.
405 static PageDesc
*page_find_alloc(tb_page_addr_t index
, int alloc
)
411 /* Level 1. Always allocated. */
412 lp
= l1_map
+ ((index
>> V_L1_SHIFT
) & (V_L1_SIZE
- 1));
415 for (i
= V_L1_SHIFT
/ V_L2_BITS
- 1; i
> 0; i
--) {
416 void **p
= atomic_rcu_read(lp
);
422 p
= g_new0(void *, V_L2_SIZE
);
423 atomic_rcu_set(lp
, p
);
426 lp
= p
+ ((index
>> (i
* V_L2_BITS
)) & (V_L2_SIZE
- 1));
429 pd
= atomic_rcu_read(lp
);
434 pd
= g_new0(PageDesc
, V_L2_SIZE
);
435 atomic_rcu_set(lp
, pd
);
438 return pd
+ (index
& (V_L2_SIZE
- 1));
441 static inline PageDesc
*page_find(tb_page_addr_t index
)
443 return page_find_alloc(index
, 0);
446 #if defined(CONFIG_USER_ONLY)
447 /* Currently it is not recommended to allocate big chunks of data in
448 user mode. It will change when a dedicated libc will be used. */
449 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
450 region in which the guest needs to run. Revisit this. */
451 #define USE_STATIC_CODE_GEN_BUFFER
454 /* Minimum size of the code gen buffer. This number is randomly chosen,
455 but not so small that we can't have a fair number of TB's live. */
456 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
458 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
459 indicated, this is constrained by the range of direct branches on the
460 host cpu, as used by the TCG implementation of goto_tb. */
461 #if defined(__x86_64__)
462 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
463 #elif defined(__sparc__)
464 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
465 #elif defined(__powerpc64__)
466 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
467 #elif defined(__powerpc__)
468 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
469 #elif defined(__aarch64__)
470 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
471 #elif defined(__arm__)
472 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
473 #elif defined(__s390x__)
474 /* We have a +- 4GB range on the branches; leave some slop. */
475 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
476 #elif defined(__mips__)
477 /* We have a 256MB branch region, but leave room to make sure the
478 main executable is also within that region. */
479 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
481 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
484 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
486 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
487 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
488 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
490 static inline size_t size_code_gen_buffer(size_t tb_size
)
492 /* Size the buffer. */
494 #ifdef USE_STATIC_CODE_GEN_BUFFER
495 tb_size
= DEFAULT_CODE_GEN_BUFFER_SIZE
;
497 /* ??? Needs adjustments. */
498 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
499 static buffer, we could size this on RESERVED_VA, on the text
500 segment size of the executable, or continue to use the default. */
501 tb_size
= (unsigned long)(ram_size
/ 4);
504 if (tb_size
< MIN_CODE_GEN_BUFFER_SIZE
) {
505 tb_size
= MIN_CODE_GEN_BUFFER_SIZE
;
507 if (tb_size
> MAX_CODE_GEN_BUFFER_SIZE
) {
508 tb_size
= MAX_CODE_GEN_BUFFER_SIZE
;
514 /* In order to use J and JAL within the code_gen_buffer, we require
515 that the buffer not cross a 256MB boundary. */
516 static inline bool cross_256mb(void *addr
, size_t size
)
518 return ((uintptr_t)addr
^ ((uintptr_t)addr
+ size
)) & ~0x0ffffffful
;
521 /* We weren't able to allocate a buffer without crossing that boundary,
522 so make do with the larger portion of the buffer that doesn't cross.
523 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
524 static inline void *split_cross_256mb(void *buf1
, size_t size1
)
526 void *buf2
= (void *)(((uintptr_t)buf1
+ size1
) & ~0x0ffffffful
);
527 size_t size2
= buf1
+ size1
- buf2
;
535 tcg_ctx
.code_gen_buffer_size
= size1
;
540 #ifdef USE_STATIC_CODE_GEN_BUFFER
541 static uint8_t static_code_gen_buffer
[DEFAULT_CODE_GEN_BUFFER_SIZE
]
542 __attribute__((aligned(CODE_GEN_ALIGN
)));
545 static inline void do_protect(void *addr
, long size
, int prot
)
548 VirtualProtect(addr
, size
, prot
, &old_protect
);
551 static inline void map_exec(void *addr
, long size
)
553 do_protect(addr
, size
, PAGE_EXECUTE_READWRITE
);
556 static inline void map_none(void *addr
, long size
)
558 do_protect(addr
, size
, PAGE_NOACCESS
);
561 static inline void do_protect(void *addr
, long size
, int prot
)
563 uintptr_t start
, end
;
565 start
= (uintptr_t)addr
;
566 start
&= qemu_real_host_page_mask
;
568 end
= (uintptr_t)addr
+ size
;
569 end
= ROUND_UP(end
, qemu_real_host_page_size
);
571 mprotect((void *)start
, end
- start
, prot
);
574 static inline void map_exec(void *addr
, long size
)
576 do_protect(addr
, size
, PROT_READ
| PROT_WRITE
| PROT_EXEC
);
579 static inline void map_none(void *addr
, long size
)
581 do_protect(addr
, size
, PROT_NONE
);
585 static inline void *alloc_code_gen_buffer(void)
587 void *buf
= static_code_gen_buffer
;
588 size_t full_size
, size
;
590 /* The size of the buffer, rounded down to end on a page boundary. */
591 full_size
= (((uintptr_t)buf
+ sizeof(static_code_gen_buffer
))
592 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
594 /* Reserve a guard page. */
595 size
= full_size
- qemu_real_host_page_size
;
597 /* Honor a command-line option limiting the size of the buffer. */
598 if (size
> tcg_ctx
.code_gen_buffer_size
) {
599 size
= (((uintptr_t)buf
+ tcg_ctx
.code_gen_buffer_size
)
600 & qemu_real_host_page_mask
) - (uintptr_t)buf
;
602 tcg_ctx
.code_gen_buffer_size
= size
;
605 if (cross_256mb(buf
, size
)) {
606 buf
= split_cross_256mb(buf
, size
);
607 size
= tcg_ctx
.code_gen_buffer_size
;
612 map_none(buf
+ size
, qemu_real_host_page_size
);
613 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
617 #elif defined(_WIN32)
618 static inline void *alloc_code_gen_buffer(void)
620 size_t size
= tcg_ctx
.code_gen_buffer_size
;
623 /* Perform the allocation in two steps, so that the guard page
624 is reserved but uncommitted. */
625 buf1
= VirtualAlloc(NULL
, size
+ qemu_real_host_page_size
,
626 MEM_RESERVE
, PAGE_NOACCESS
);
628 buf2
= VirtualAlloc(buf1
, size
, MEM_COMMIT
, PAGE_EXECUTE_READWRITE
);
629 assert(buf1
== buf2
);
635 static inline void *alloc_code_gen_buffer(void)
637 int flags
= MAP_PRIVATE
| MAP_ANONYMOUS
;
639 size_t size
= tcg_ctx
.code_gen_buffer_size
;
642 /* Constrain the position of the buffer based on the host cpu.
643 Note that these addresses are chosen in concert with the
644 addresses assigned in the relevant linker script file. */
645 # if defined(__PIE__) || defined(__PIC__)
646 /* Don't bother setting a preferred location if we're building
647 a position-independent executable. We're more likely to get
648 an address near the main executable if we let the kernel
649 choose the address. */
650 # elif defined(__x86_64__) && defined(MAP_32BIT)
651 /* Force the memory down into low memory with the executable.
652 Leave the choice of exact location with the kernel. */
654 /* Cannot expect to map more than 800MB in low memory. */
655 if (size
> 800u * 1024 * 1024) {
656 tcg_ctx
.code_gen_buffer_size
= size
= 800u * 1024 * 1024;
658 # elif defined(__sparc__)
659 start
= 0x40000000ul
;
660 # elif defined(__s390x__)
661 start
= 0x90000000ul
;
662 # elif defined(__mips__)
663 # if _MIPS_SIM == _ABI64
664 start
= 0x128000000ul
;
666 start
= 0x08000000ul
;
670 buf
= mmap((void *)start
, size
+ qemu_real_host_page_size
,
671 PROT_NONE
, flags
, -1, 0);
672 if (buf
== MAP_FAILED
) {
677 if (cross_256mb(buf
, size
)) {
678 /* Try again, with the original still mapped, to avoid re-acquiring
679 that 256mb crossing. This time don't specify an address. */
681 void *buf2
= mmap(NULL
, size
+ qemu_real_host_page_size
,
682 PROT_NONE
, flags
, -1, 0);
683 switch (buf2
!= MAP_FAILED
) {
685 if (!cross_256mb(buf2
, size
)) {
686 /* Success! Use the new buffer. */
687 munmap(buf
, size
+ qemu_real_host_page_size
);
690 /* Failure. Work with what we had. */
691 munmap(buf2
, size
+ qemu_real_host_page_size
);
694 /* Split the original buffer. Free the smaller half. */
695 buf2
= split_cross_256mb(buf
, size
);
696 size2
= tcg_ctx
.code_gen_buffer_size
;
698 munmap(buf
+ size2
+ qemu_real_host_page_size
, size
- size2
);
700 munmap(buf
, size
- size2
);
709 /* Make the final buffer accessible. The guard page at the end
710 will remain inaccessible with PROT_NONE. */
711 mprotect(buf
, size
, PROT_WRITE
| PROT_READ
| PROT_EXEC
);
713 /* Request large pages for the buffer. */
714 qemu_madvise(buf
, size
, QEMU_MADV_HUGEPAGE
);
718 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
720 static inline void code_gen_alloc(size_t tb_size
)
722 tcg_ctx
.code_gen_buffer_size
= size_code_gen_buffer(tb_size
);
723 tcg_ctx
.code_gen_buffer
= alloc_code_gen_buffer();
724 if (tcg_ctx
.code_gen_buffer
== NULL
) {
725 fprintf(stderr
, "Could not allocate dynamic translator buffer\n");
729 /* Estimate a good size for the number of TBs we can support. We
730 still haven't deducted the prologue from the buffer size here,
731 but that's minimal and won't affect the estimate much. */
732 tcg_ctx
.code_gen_max_blocks
733 = tcg_ctx
.code_gen_buffer_size
/ CODE_GEN_AVG_BLOCK_SIZE
;
734 tcg_ctx
.tb_ctx
.tbs
= g_new(TranslationBlock
, tcg_ctx
.code_gen_max_blocks
);
736 qemu_mutex_init(&tcg_ctx
.tb_ctx
.tb_lock
);
739 /* Must be called before using the QEMU cpus. 'tb_size' is the size
740 (in bytes) allocated to the translation buffer. Zero means default
742 void tcg_exec_init(unsigned long tb_size
)
746 code_gen_alloc(tb_size
);
747 #if defined(CONFIG_SOFTMMU)
748 /* There's no guest base to take into account, so go ahead and
749 initialize the prologue now. */
750 tcg_prologue_init(&tcg_ctx
);
754 bool tcg_enabled(void)
756 return tcg_ctx
.code_gen_buffer
!= NULL
;
759 /* Allocate a new translation block. Flush the translation buffer if
760 too many translation blocks or too much generated code. */
761 static TranslationBlock
*tb_alloc(target_ulong pc
)
763 TranslationBlock
*tb
;
765 if (tcg_ctx
.tb_ctx
.nb_tbs
>= tcg_ctx
.code_gen_max_blocks
) {
768 tb
= &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
++];
774 void tb_free(TranslationBlock
*tb
)
776 /* In practice this is mostly used for single use temporary TB
777 Ignore the hard cases and just back up if this TB happens to
778 be the last one generated. */
779 if (tcg_ctx
.tb_ctx
.nb_tbs
> 0 &&
780 tb
== &tcg_ctx
.tb_ctx
.tbs
[tcg_ctx
.tb_ctx
.nb_tbs
- 1]) {
781 tcg_ctx
.code_gen_ptr
= tb
->tc_ptr
;
782 tcg_ctx
.tb_ctx
.nb_tbs
--;
786 static inline void invalidate_page_bitmap(PageDesc
*p
)
788 g_free(p
->code_bitmap
);
789 p
->code_bitmap
= NULL
;
790 p
->code_write_count
= 0;
793 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
794 static void page_flush_tb_1(int level
, void **lp
)
804 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
805 pd
[i
].first_tb
= NULL
;
806 invalidate_page_bitmap(pd
+ i
);
811 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
812 page_flush_tb_1(level
- 1, pp
+ i
);
817 static void page_flush_tb(void)
821 for (i
= 0; i
< V_L1_SIZE
; i
++) {
822 page_flush_tb_1(V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
826 /* flush all the translation blocks */
827 /* XXX: tb_flush is currently not thread safe */
828 void tb_flush(CPUState
*cpu
)
830 #if defined(DEBUG_FLUSH)
831 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
832 (unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
),
833 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.tb_ctx
.nb_tbs
> 0 ?
834 ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)) /
835 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
837 if ((unsigned long)(tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
)
838 > tcg_ctx
.code_gen_buffer_size
) {
839 cpu_abort(cpu
, "Internal error: code buffer overflow\n");
841 tcg_ctx
.tb_ctx
.nb_tbs
= 0;
844 memset(cpu
->tb_jmp_cache
, 0, sizeof(cpu
->tb_jmp_cache
));
847 memset(tcg_ctx
.tb_ctx
.tb_phys_hash
, 0, sizeof(tcg_ctx
.tb_ctx
.tb_phys_hash
));
850 tcg_ctx
.code_gen_ptr
= tcg_ctx
.code_gen_buffer
;
851 /* XXX: flush processor icache at this point if cache flush is
853 tcg_ctx
.tb_ctx
.tb_flush_count
++;
856 #ifdef DEBUG_TB_CHECK
858 static void tb_invalidate_check(target_ulong address
)
860 TranslationBlock
*tb
;
863 address
&= TARGET_PAGE_MASK
;
864 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
865 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
866 tb
= tb
->phys_hash_next
) {
867 if (!(address
+ TARGET_PAGE_SIZE
<= tb
->pc
||
868 address
>= tb
->pc
+ tb
->size
)) {
869 printf("ERROR invalidate: address=" TARGET_FMT_lx
870 " PC=%08lx size=%04x\n",
871 address
, (long)tb
->pc
, tb
->size
);
877 /* verify that all the pages have correct rights for code */
878 static void tb_page_check(void)
880 TranslationBlock
*tb
;
881 int i
, flags1
, flags2
;
883 for (i
= 0; i
< CODE_GEN_PHYS_HASH_SIZE
; i
++) {
884 for (tb
= tcg_ctx
.tb_ctx
.tb_phys_hash
[i
]; tb
!= NULL
;
885 tb
= tb
->phys_hash_next
) {
886 flags1
= page_get_flags(tb
->pc
);
887 flags2
= page_get_flags(tb
->pc
+ tb
->size
- 1);
888 if ((flags1
& PAGE_WRITE
) || (flags2
& PAGE_WRITE
)) {
889 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
890 (long)tb
->pc
, tb
->size
, flags1
, flags2
);
898 static inline void tb_hash_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
900 TranslationBlock
*tb1
;
905 *ptb
= tb1
->phys_hash_next
;
908 ptb
= &tb1
->phys_hash_next
;
912 static inline void tb_page_remove(TranslationBlock
**ptb
, TranslationBlock
*tb
)
914 TranslationBlock
*tb1
;
919 n1
= (uintptr_t)tb1
& 3;
920 tb1
= (TranslationBlock
*)((uintptr_t)tb1
& ~3);
922 *ptb
= tb1
->page_next
[n1
];
925 ptb
= &tb1
->page_next
[n1
];
929 static inline void tb_jmp_remove(TranslationBlock
*tb
, int n
)
931 TranslationBlock
*tb1
;
935 ptb
= &tb
->jmp_list_next
[n
];
937 /* find tb(n) in circular list */
941 tb1
= (TranslationBlock
*)(ntb
& ~3);
942 if (n1
== n
&& tb1
== tb
) {
946 ptb
= &tb1
->jmp_list_first
;
948 ptb
= &tb1
->jmp_list_next
[n1
];
951 /* now we can suppress tb(n) from the list */
952 *ptb
= tb
->jmp_list_next
[n
];
954 tb
->jmp_list_next
[n
] = (uintptr_t)NULL
;
958 /* reset the jump entry 'n' of a TB so that it is not chained to
960 static inline void tb_reset_jump(TranslationBlock
*tb
, int n
)
962 uintptr_t addr
= (uintptr_t)(tb
->tc_ptr
+ tb
->jmp_reset_offset
[n
]);
963 tb_set_jmp_target(tb
, n
, addr
);
966 /* invalidate one TB */
967 void tb_phys_invalidate(TranslationBlock
*tb
, tb_page_addr_t page_addr
)
972 tb_page_addr_t phys_pc
;
975 /* remove the TB from the hash list */
976 phys_pc
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
977 h
= tb_phys_hash_func(phys_pc
);
978 tb_hash_remove(&tcg_ctx
.tb_ctx
.tb_phys_hash
[h
], tb
);
980 /* remove the TB from the page list */
981 if (tb
->page_addr
[0] != page_addr
) {
982 p
= page_find(tb
->page_addr
[0] >> TARGET_PAGE_BITS
);
983 tb_page_remove(&p
->first_tb
, tb
);
984 invalidate_page_bitmap(p
);
986 if (tb
->page_addr
[1] != -1 && tb
->page_addr
[1] != page_addr
) {
987 p
= page_find(tb
->page_addr
[1] >> TARGET_PAGE_BITS
);
988 tb_page_remove(&p
->first_tb
, tb
);
989 invalidate_page_bitmap(p
);
992 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
994 /* remove the TB from the hash list */
995 h
= tb_jmp_cache_hash_func(tb
->pc
);
997 if (cpu
->tb_jmp_cache
[h
] == tb
) {
998 cpu
->tb_jmp_cache
[h
] = NULL
;
1002 /* suppress this TB from the two jump lists */
1003 tb_jmp_remove(tb
, 0);
1004 tb_jmp_remove(tb
, 1);
1006 /* suppress any remaining jumps to this TB */
1007 tb1
= tb
->jmp_list_first
;
1009 TranslationBlock
*tmp_tb
;
1014 tmp_tb
= (TranslationBlock
*)(tb1
& ~3);
1015 tb2
= tmp_tb
->jmp_list_next
[n1
];
1016 tb_reset_jump(tmp_tb
, n1
);
1017 tmp_tb
->jmp_list_next
[n1
] = (uintptr_t)NULL
;
1021 assert(((uintptr_t)tb
& 3) == 0);
1022 tb
->jmp_list_first
= (uintptr_t)tb
| 2; /* fail safe */
1024 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
++;
1027 static void build_page_bitmap(PageDesc
*p
)
1029 int n
, tb_start
, tb_end
;
1030 TranslationBlock
*tb
;
1032 p
->code_bitmap
= bitmap_new(TARGET_PAGE_SIZE
);
1035 while (tb
!= NULL
) {
1036 n
= (uintptr_t)tb
& 3;
1037 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1038 /* NOTE: this is subtle as a TB may span two physical pages */
1040 /* NOTE: tb_end may be after the end of the page, but
1041 it is not a problem */
1042 tb_start
= tb
->pc
& ~TARGET_PAGE_MASK
;
1043 tb_end
= tb_start
+ tb
->size
;
1044 if (tb_end
> TARGET_PAGE_SIZE
) {
1045 tb_end
= TARGET_PAGE_SIZE
;
1049 tb_end
= ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1051 bitmap_set(p
->code_bitmap
, tb_start
, tb_end
- tb_start
);
1052 tb
= tb
->page_next
[n
];
1056 /* Called with mmap_lock held for user mode emulation. */
1057 TranslationBlock
*tb_gen_code(CPUState
*cpu
,
1058 target_ulong pc
, target_ulong cs_base
,
1059 uint32_t flags
, int cflags
)
1061 CPUArchState
*env
= cpu
->env_ptr
;
1062 TranslationBlock
*tb
;
1063 tb_page_addr_t phys_pc
, phys_page2
;
1064 target_ulong virt_page2
;
1065 tcg_insn_unit
*gen_code_buf
;
1066 int gen_code_size
, search_size
;
1067 #ifdef CONFIG_PROFILER
1071 phys_pc
= get_page_addr_code(env
, pc
);
1072 if (use_icount
&& !(cflags
& CF_IGNORE_ICOUNT
)) {
1073 cflags
|= CF_USE_ICOUNT
;
1077 if (unlikely(!tb
)) {
1079 /* flush must be done */
1081 /* cannot fail at this point */
1084 /* Don't forget to invalidate previous TB info. */
1085 tcg_ctx
.tb_ctx
.tb_invalidated_flag
= 1;
1088 gen_code_buf
= tcg_ctx
.code_gen_ptr
;
1089 tb
->tc_ptr
= gen_code_buf
;
1090 tb
->cs_base
= cs_base
;
1092 tb
->cflags
= cflags
;
1094 #ifdef CONFIG_PROFILER
1095 tcg_ctx
.tb_count1
++; /* includes aborted translations because of
1097 ti
= profile_getclock();
1100 tcg_func_start(&tcg_ctx
);
1102 gen_intermediate_code(env
, tb
);
1104 trace_translate_block(tb
, tb
->pc
, tb
->tc_ptr
);
1106 /* generate machine code */
1107 tb
->jmp_reset_offset
[0] = TB_JMP_RESET_OFFSET_INVALID
;
1108 tb
->jmp_reset_offset
[1] = TB_JMP_RESET_OFFSET_INVALID
;
1109 tcg_ctx
.tb_jmp_reset_offset
= tb
->jmp_reset_offset
;
1110 #ifdef USE_DIRECT_JUMP
1111 tcg_ctx
.tb_jmp_insn_offset
= tb
->jmp_insn_offset
;
1112 tcg_ctx
.tb_jmp_target_addr
= NULL
;
1114 tcg_ctx
.tb_jmp_insn_offset
= NULL
;
1115 tcg_ctx
.tb_jmp_target_addr
= tb
->jmp_target_addr
;
1118 #ifdef CONFIG_PROFILER
1120 tcg_ctx
.interm_time
+= profile_getclock() - ti
;
1121 tcg_ctx
.code_time
-= profile_getclock();
1124 /* ??? Overflow could be handled better here. In particular, we
1125 don't need to re-do gen_intermediate_code, nor should we re-do
1126 the tcg optimization currently hidden inside tcg_gen_code. All
1127 that should be required is to flush the TBs, allocate a new TB,
1128 re-initialize it per above, and re-do the actual code generation. */
1129 gen_code_size
= tcg_gen_code(&tcg_ctx
, tb
);
1130 if (unlikely(gen_code_size
< 0)) {
1131 goto buffer_overflow
;
1133 search_size
= encode_search(tb
, (void *)gen_code_buf
+ gen_code_size
);
1134 if (unlikely(search_size
< 0)) {
1135 goto buffer_overflow
;
1138 #ifdef CONFIG_PROFILER
1139 tcg_ctx
.code_time
+= profile_getclock();
1140 tcg_ctx
.code_in_len
+= tb
->size
;
1141 tcg_ctx
.code_out_len
+= gen_code_size
;
1142 tcg_ctx
.search_out_len
+= search_size
;
1146 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM
) &&
1147 qemu_log_in_addr_range(tb
->pc
)) {
1148 qemu_log("OUT: [size=%d]\n", gen_code_size
);
1149 log_disas(tb
->tc_ptr
, gen_code_size
);
1155 tcg_ctx
.code_gen_ptr
= (void *)
1156 ROUND_UP((uintptr_t)gen_code_buf
+ gen_code_size
+ search_size
,
1159 /* check next page if needed */
1160 virt_page2
= (pc
+ tb
->size
- 1) & TARGET_PAGE_MASK
;
1162 if ((pc
& TARGET_PAGE_MASK
) != virt_page2
) {
1163 phys_page2
= get_page_addr_code(env
, virt_page2
);
1165 tb_link_page(tb
, phys_pc
, phys_page2
);
1170 * Invalidate all TBs which intersect with the target physical address range
1171 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1172 * 'is_cpu_write_access' should be true if called from a real cpu write
1173 * access: the virtual CPU will exit the current TB if code is modified inside
1176 * Called with mmap_lock held for user-mode emulation
1178 void tb_invalidate_phys_range(tb_page_addr_t start
, tb_page_addr_t end
)
1180 while (start
< end
) {
1181 tb_invalidate_phys_page_range(start
, end
, 0);
1182 start
&= TARGET_PAGE_MASK
;
1183 start
+= TARGET_PAGE_SIZE
;
1188 * Invalidate all TBs which intersect with the target physical address range
1189 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1190 * 'is_cpu_write_access' should be true if called from a real cpu write
1191 * access: the virtual CPU will exit the current TB if code is modified inside
1194 * Called with mmap_lock held for user-mode emulation
1196 void tb_invalidate_phys_page_range(tb_page_addr_t start
, tb_page_addr_t end
,
1197 int is_cpu_write_access
)
1199 TranslationBlock
*tb
, *tb_next
, *saved_tb
;
1200 CPUState
*cpu
= current_cpu
;
1201 #if defined(TARGET_HAS_PRECISE_SMC)
1202 CPUArchState
*env
= NULL
;
1204 tb_page_addr_t tb_start
, tb_end
;
1207 #ifdef TARGET_HAS_PRECISE_SMC
1208 int current_tb_not_found
= is_cpu_write_access
;
1209 TranslationBlock
*current_tb
= NULL
;
1210 int current_tb_modified
= 0;
1211 target_ulong current_pc
= 0;
1212 target_ulong current_cs_base
= 0;
1213 uint32_t current_flags
= 0;
1214 #endif /* TARGET_HAS_PRECISE_SMC */
1216 p
= page_find(start
>> TARGET_PAGE_BITS
);
1220 #if defined(TARGET_HAS_PRECISE_SMC)
1226 /* we remove all the TBs in the range [start, end[ */
1227 /* XXX: see if in some cases it could be faster to invalidate all
1230 while (tb
!= NULL
) {
1231 n
= (uintptr_t)tb
& 3;
1232 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1233 tb_next
= tb
->page_next
[n
];
1234 /* NOTE: this is subtle as a TB may span two physical pages */
1236 /* NOTE: tb_end may be after the end of the page, but
1237 it is not a problem */
1238 tb_start
= tb
->page_addr
[0] + (tb
->pc
& ~TARGET_PAGE_MASK
);
1239 tb_end
= tb_start
+ tb
->size
;
1241 tb_start
= tb
->page_addr
[1];
1242 tb_end
= tb_start
+ ((tb
->pc
+ tb
->size
) & ~TARGET_PAGE_MASK
);
1244 if (!(tb_end
<= start
|| tb_start
>= end
)) {
1245 #ifdef TARGET_HAS_PRECISE_SMC
1246 if (current_tb_not_found
) {
1247 current_tb_not_found
= 0;
1249 if (cpu
->mem_io_pc
) {
1250 /* now we have a real cpu fault */
1251 current_tb
= tb_find_pc(cpu
->mem_io_pc
);
1254 if (current_tb
== tb
&&
1255 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1256 /* If we are modifying the current TB, we must stop
1257 its execution. We could be more precise by checking
1258 that the modification is after the current PC, but it
1259 would require a specialized function to partially
1260 restore the CPU state */
1262 current_tb_modified
= 1;
1263 cpu_restore_state_from_tb(cpu
, current_tb
, cpu
->mem_io_pc
);
1264 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1267 #endif /* TARGET_HAS_PRECISE_SMC */
1268 /* we need to do that to handle the case where a signal
1269 occurs while doing tb_phys_invalidate() */
1272 saved_tb
= cpu
->current_tb
;
1273 cpu
->current_tb
= NULL
;
1275 tb_phys_invalidate(tb
, -1);
1277 cpu
->current_tb
= saved_tb
;
1278 if (cpu
->interrupt_request
&& cpu
->current_tb
) {
1279 cpu_interrupt(cpu
, cpu
->interrupt_request
);
1285 #if !defined(CONFIG_USER_ONLY)
1286 /* if no code remaining, no need to continue to use slow writes */
1288 invalidate_page_bitmap(p
);
1289 tlb_unprotect_code(start
);
1292 #ifdef TARGET_HAS_PRECISE_SMC
1293 if (current_tb_modified
) {
1294 /* we generate a block containing just the instruction
1295 modifying the memory. It will ensure that it cannot modify
1297 cpu
->current_tb
= NULL
;
1298 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1299 cpu_resume_from_signal(cpu
, NULL
);
1304 /* len must be <= 8 and start must be a multiple of len */
1305 void tb_invalidate_phys_page_fast(tb_page_addr_t start
, int len
)
1311 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1312 cpu_single_env
->mem_io_vaddr
, len
,
1313 cpu_single_env
->eip
,
1314 cpu_single_env
->eip
+
1315 (intptr_t)cpu_single_env
->segs
[R_CS
].base
);
1318 p
= page_find(start
>> TARGET_PAGE_BITS
);
1322 if (!p
->code_bitmap
&&
1323 ++p
->code_write_count
>= SMC_BITMAP_USE_THRESHOLD
) {
1324 /* build code bitmap */
1325 build_page_bitmap(p
);
1327 if (p
->code_bitmap
) {
1331 nr
= start
& ~TARGET_PAGE_MASK
;
1332 b
= p
->code_bitmap
[BIT_WORD(nr
)] >> (nr
& (BITS_PER_LONG
- 1));
1333 if (b
& ((1 << len
) - 1)) {
1338 tb_invalidate_phys_page_range(start
, start
+ len
, 1);
1342 #if !defined(CONFIG_SOFTMMU)
1343 /* Called with mmap_lock held. */
1344 static void tb_invalidate_phys_page(tb_page_addr_t addr
,
1345 uintptr_t pc
, void *puc
,
1348 TranslationBlock
*tb
;
1351 #ifdef TARGET_HAS_PRECISE_SMC
1352 TranslationBlock
*current_tb
= NULL
;
1353 CPUState
*cpu
= current_cpu
;
1354 CPUArchState
*env
= NULL
;
1355 int current_tb_modified
= 0;
1356 target_ulong current_pc
= 0;
1357 target_ulong current_cs_base
= 0;
1358 uint32_t current_flags
= 0;
1361 addr
&= TARGET_PAGE_MASK
;
1362 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1367 #ifdef TARGET_HAS_PRECISE_SMC
1368 if (tb
&& pc
!= 0) {
1369 current_tb
= tb_find_pc(pc
);
1375 while (tb
!= NULL
) {
1376 n
= (uintptr_t)tb
& 3;
1377 tb
= (TranslationBlock
*)((uintptr_t)tb
& ~3);
1378 #ifdef TARGET_HAS_PRECISE_SMC
1379 if (current_tb
== tb
&&
1380 (current_tb
->cflags
& CF_COUNT_MASK
) != 1) {
1381 /* If we are modifying the current TB, we must stop
1382 its execution. We could be more precise by checking
1383 that the modification is after the current PC, but it
1384 would require a specialized function to partially
1385 restore the CPU state */
1387 current_tb_modified
= 1;
1388 cpu_restore_state_from_tb(cpu
, current_tb
, pc
);
1389 cpu_get_tb_cpu_state(env
, ¤t_pc
, ¤t_cs_base
,
1392 #endif /* TARGET_HAS_PRECISE_SMC */
1393 tb_phys_invalidate(tb
, addr
);
1394 tb
= tb
->page_next
[n
];
1397 #ifdef TARGET_HAS_PRECISE_SMC
1398 if (current_tb_modified
) {
1399 /* we generate a block containing just the instruction
1400 modifying the memory. It will ensure that it cannot modify
1402 cpu
->current_tb
= NULL
;
1403 tb_gen_code(cpu
, current_pc
, current_cs_base
, current_flags
, 1);
1407 cpu_resume_from_signal(cpu
, puc
);
1413 /* add the tb in the target page and protect it if necessary
1415 * Called with mmap_lock held for user-mode emulation.
1417 static inline void tb_alloc_page(TranslationBlock
*tb
,
1418 unsigned int n
, tb_page_addr_t page_addr
)
1421 #ifndef CONFIG_USER_ONLY
1422 bool page_already_protected
;
1425 tb
->page_addr
[n
] = page_addr
;
1426 p
= page_find_alloc(page_addr
>> TARGET_PAGE_BITS
, 1);
1427 tb
->page_next
[n
] = p
->first_tb
;
1428 #ifndef CONFIG_USER_ONLY
1429 page_already_protected
= p
->first_tb
!= NULL
;
1431 p
->first_tb
= (TranslationBlock
*)((uintptr_t)tb
| n
);
1432 invalidate_page_bitmap(p
);
1434 #if defined(CONFIG_USER_ONLY)
1435 if (p
->flags
& PAGE_WRITE
) {
1440 /* force the host page as non writable (writes will have a
1441 page fault + mprotect overhead) */
1442 page_addr
&= qemu_host_page_mask
;
1444 for (addr
= page_addr
; addr
< page_addr
+ qemu_host_page_size
;
1445 addr
+= TARGET_PAGE_SIZE
) {
1447 p2
= page_find(addr
>> TARGET_PAGE_BITS
);
1452 p2
->flags
&= ~PAGE_WRITE
;
1454 mprotect(g2h(page_addr
), qemu_host_page_size
,
1455 (prot
& PAGE_BITS
) & ~PAGE_WRITE
);
1456 #ifdef DEBUG_TB_INVALIDATE
1457 printf("protecting code page: 0x" TARGET_FMT_lx
"\n",
1462 /* if some code is already present, then the pages are already
1463 protected. So we handle the case where only the first TB is
1464 allocated in a physical page */
1465 if (!page_already_protected
) {
1466 tlb_protect_code(page_addr
);
1471 /* add a new TB and link it to the physical page tables. phys_page2 is
1472 * (-1) to indicate that only one page contains the TB.
1474 * Called with mmap_lock held for user-mode emulation.
1476 static void tb_link_page(TranslationBlock
*tb
, tb_page_addr_t phys_pc
,
1477 tb_page_addr_t phys_page2
)
1480 TranslationBlock
**ptb
;
1482 /* add in the physical hash table */
1483 h
= tb_phys_hash_func(phys_pc
);
1484 ptb
= &tcg_ctx
.tb_ctx
.tb_phys_hash
[h
];
1485 tb
->phys_hash_next
= *ptb
;
1488 /* add in the page list */
1489 tb_alloc_page(tb
, 0, phys_pc
& TARGET_PAGE_MASK
);
1490 if (phys_page2
!= -1) {
1491 tb_alloc_page(tb
, 1, phys_page2
);
1493 tb
->page_addr
[1] = -1;
1496 assert(((uintptr_t)tb
& 3) == 0);
1497 tb
->jmp_list_first
= (uintptr_t)tb
| 2;
1498 tb
->jmp_list_next
[0] = (uintptr_t)NULL
;
1499 tb
->jmp_list_next
[1] = (uintptr_t)NULL
;
1501 /* init original jump addresses */
1502 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1503 tb_reset_jump(tb
, 0);
1505 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1506 tb_reset_jump(tb
, 1);
1509 #ifdef DEBUG_TB_CHECK
1514 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1515 tb[1].tc_ptr. Return NULL if not found */
1516 static TranslationBlock
*tb_find_pc(uintptr_t tc_ptr
)
1518 int m_min
, m_max
, m
;
1520 TranslationBlock
*tb
;
1522 if (tcg_ctx
.tb_ctx
.nb_tbs
<= 0) {
1525 if (tc_ptr
< (uintptr_t)tcg_ctx
.code_gen_buffer
||
1526 tc_ptr
>= (uintptr_t)tcg_ctx
.code_gen_ptr
) {
1529 /* binary search (cf Knuth) */
1531 m_max
= tcg_ctx
.tb_ctx
.nb_tbs
- 1;
1532 while (m_min
<= m_max
) {
1533 m
= (m_min
+ m_max
) >> 1;
1534 tb
= &tcg_ctx
.tb_ctx
.tbs
[m
];
1535 v
= (uintptr_t)tb
->tc_ptr
;
1538 } else if (tc_ptr
< v
) {
1544 return &tcg_ctx
.tb_ctx
.tbs
[m_max
];
1547 #if !defined(CONFIG_USER_ONLY)
1548 void tb_invalidate_phys_addr(AddressSpace
*as
, hwaddr addr
)
1550 ram_addr_t ram_addr
;
1555 mr
= address_space_translate(as
, addr
, &addr
, &l
, false);
1556 if (!(memory_region_is_ram(mr
)
1557 || memory_region_is_romd(mr
))) {
1561 ram_addr
= (memory_region_get_ram_addr(mr
) & TARGET_PAGE_MASK
)
1563 tb_invalidate_phys_page_range(ram_addr
, ram_addr
+ 1, 0);
1566 #endif /* !defined(CONFIG_USER_ONLY) */
1568 void tb_check_watchpoint(CPUState
*cpu
)
1570 TranslationBlock
*tb
;
1572 tb
= tb_find_pc(cpu
->mem_io_pc
);
1574 /* We can use retranslation to find the PC. */
1575 cpu_restore_state_from_tb(cpu
, tb
, cpu
->mem_io_pc
);
1576 tb_phys_invalidate(tb
, -1);
1578 /* The exception probably happened in a helper. The CPU state should
1579 have been saved before calling it. Fetch the PC from there. */
1580 CPUArchState
*env
= cpu
->env_ptr
;
1581 target_ulong pc
, cs_base
;
1582 tb_page_addr_t addr
;
1585 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
1586 addr
= get_page_addr_code(env
, pc
);
1587 tb_invalidate_phys_range(addr
, addr
+ 1);
1591 #ifndef CONFIG_USER_ONLY
1592 /* in deterministic execution mode, instructions doing device I/Os
1593 must be at the end of the TB */
1594 void cpu_io_recompile(CPUState
*cpu
, uintptr_t retaddr
)
1596 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1597 CPUArchState
*env
= cpu
->env_ptr
;
1599 TranslationBlock
*tb
;
1601 target_ulong pc
, cs_base
;
1604 tb
= tb_find_pc(retaddr
);
1606 cpu_abort(cpu
, "cpu_io_recompile: could not find TB for pc=%p",
1609 n
= cpu
->icount_decr
.u16
.low
+ tb
->icount
;
1610 cpu_restore_state_from_tb(cpu
, tb
, retaddr
);
1611 /* Calculate how many instructions had been executed before the fault
1613 n
= n
- cpu
->icount_decr
.u16
.low
;
1614 /* Generate a new TB ending on the I/O insn. */
1616 /* On MIPS and SH, delay slot instructions can only be restarted if
1617 they were already the first instruction in the TB. If this is not
1618 the first instruction in a TB then re-execute the preceding
1620 #if defined(TARGET_MIPS)
1621 if ((env
->hflags
& MIPS_HFLAG_BMASK
) != 0 && n
> 1) {
1622 env
->active_tc
.PC
-= (env
->hflags
& MIPS_HFLAG_B16
? 2 : 4);
1623 cpu
->icount_decr
.u16
.low
++;
1624 env
->hflags
&= ~MIPS_HFLAG_BMASK
;
1626 #elif defined(TARGET_SH4)
1627 if ((env
->flags
& ((DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
))) != 0
1630 cpu
->icount_decr
.u16
.low
++;
1631 env
->flags
&= ~(DELAY_SLOT
| DELAY_SLOT_CONDITIONAL
);
1634 /* This should never happen. */
1635 if (n
> CF_COUNT_MASK
) {
1636 cpu_abort(cpu
, "TB too big during recompile");
1639 cflags
= n
| CF_LAST_IO
;
1641 cs_base
= tb
->cs_base
;
1643 tb_phys_invalidate(tb
, -1);
1644 if (tb
->cflags
& CF_NOCACHE
) {
1646 /* Invalidate original TB if this TB was generated in
1647 * cpu_exec_nocache() */
1648 tb_phys_invalidate(tb
->orig_tb
, -1);
1652 /* FIXME: In theory this could raise an exception. In practice
1653 we have already translated the block once so it's probably ok. */
1654 tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
1655 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1656 the first in the TB) then we end up generating a whole new TB and
1657 repeating the fault, which is horribly inefficient.
1658 Better would be to execute just this insn uncached, or generate a
1660 cpu_resume_from_signal(cpu
, NULL
);
1663 void tb_flush_jmp_cache(CPUState
*cpu
, target_ulong addr
)
1667 /* Discard jump cache entries for any tb which might potentially
1668 overlap the flushed page. */
1669 i
= tb_jmp_cache_hash_page(addr
- TARGET_PAGE_SIZE
);
1670 memset(&cpu
->tb_jmp_cache
[i
], 0,
1671 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1673 i
= tb_jmp_cache_hash_page(addr
);
1674 memset(&cpu
->tb_jmp_cache
[i
], 0,
1675 TB_JMP_PAGE_SIZE
* sizeof(TranslationBlock
*));
1678 void dump_exec_info(FILE *f
, fprintf_function cpu_fprintf
)
1680 int i
, target_code_size
, max_target_code_size
;
1681 int direct_jmp_count
, direct_jmp2_count
, cross_page
;
1682 TranslationBlock
*tb
;
1684 target_code_size
= 0;
1685 max_target_code_size
= 0;
1687 direct_jmp_count
= 0;
1688 direct_jmp2_count
= 0;
1689 for (i
= 0; i
< tcg_ctx
.tb_ctx
.nb_tbs
; i
++) {
1690 tb
= &tcg_ctx
.tb_ctx
.tbs
[i
];
1691 target_code_size
+= tb
->size
;
1692 if (tb
->size
> max_target_code_size
) {
1693 max_target_code_size
= tb
->size
;
1695 if (tb
->page_addr
[1] != -1) {
1698 if (tb
->jmp_reset_offset
[0] != TB_JMP_RESET_OFFSET_INVALID
) {
1700 if (tb
->jmp_reset_offset
[1] != TB_JMP_RESET_OFFSET_INVALID
) {
1701 direct_jmp2_count
++;
1705 /* XXX: avoid using doubles ? */
1706 cpu_fprintf(f
, "Translation buffer state:\n");
1707 cpu_fprintf(f
, "gen code size %td/%zd\n",
1708 tcg_ctx
.code_gen_ptr
- tcg_ctx
.code_gen_buffer
,
1709 tcg_ctx
.code_gen_highwater
- tcg_ctx
.code_gen_buffer
);
1710 cpu_fprintf(f
, "TB count %d/%d\n",
1711 tcg_ctx
.tb_ctx
.nb_tbs
, tcg_ctx
.code_gen_max_blocks
);
1712 cpu_fprintf(f
, "TB avg target size %d max=%d bytes\n",
1713 tcg_ctx
.tb_ctx
.nb_tbs
? target_code_size
/
1714 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1715 max_target_code_size
);
1716 cpu_fprintf(f
, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1717 tcg_ctx
.tb_ctx
.nb_tbs
? (tcg_ctx
.code_gen_ptr
-
1718 tcg_ctx
.code_gen_buffer
) /
1719 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1720 target_code_size
? (double) (tcg_ctx
.code_gen_ptr
-
1721 tcg_ctx
.code_gen_buffer
) /
1722 target_code_size
: 0);
1723 cpu_fprintf(f
, "cross page TB count %d (%d%%)\n", cross_page
,
1724 tcg_ctx
.tb_ctx
.nb_tbs
? (cross_page
* 100) /
1725 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1726 cpu_fprintf(f
, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1728 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp_count
* 100) /
1729 tcg_ctx
.tb_ctx
.nb_tbs
: 0,
1731 tcg_ctx
.tb_ctx
.nb_tbs
? (direct_jmp2_count
* 100) /
1732 tcg_ctx
.tb_ctx
.nb_tbs
: 0);
1733 cpu_fprintf(f
, "\nStatistics:\n");
1734 cpu_fprintf(f
, "TB flush count %d\n", tcg_ctx
.tb_ctx
.tb_flush_count
);
1735 cpu_fprintf(f
, "TB invalidate count %d\n",
1736 tcg_ctx
.tb_ctx
.tb_phys_invalidate_count
);
1737 cpu_fprintf(f
, "TLB flush count %d\n", tlb_flush_count
);
1738 tcg_dump_info(f
, cpu_fprintf
);
1741 void dump_opcount_info(FILE *f
, fprintf_function cpu_fprintf
)
1743 tcg_dump_op_count(f
, cpu_fprintf
);
1746 #else /* CONFIG_USER_ONLY */
1748 void cpu_interrupt(CPUState
*cpu
, int mask
)
1750 cpu
->interrupt_request
|= mask
;
1751 cpu
->tcg_exit_req
= 1;
1755 * Walks guest process memory "regions" one by one
1756 * and calls callback function 'fn' for each region.
1758 struct walk_memory_regions_data
{
1759 walk_memory_regions_fn fn
;
1765 static int walk_memory_regions_end(struct walk_memory_regions_data
*data
,
1766 target_ulong end
, int new_prot
)
1768 if (data
->start
!= -1u) {
1769 int rc
= data
->fn(data
->priv
, data
->start
, end
, data
->prot
);
1775 data
->start
= (new_prot
? end
: -1u);
1776 data
->prot
= new_prot
;
1781 static int walk_memory_regions_1(struct walk_memory_regions_data
*data
,
1782 target_ulong base
, int level
, void **lp
)
1788 return walk_memory_regions_end(data
, base
, 0);
1794 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1795 int prot
= pd
[i
].flags
;
1797 pa
= base
| (i
<< TARGET_PAGE_BITS
);
1798 if (prot
!= data
->prot
) {
1799 rc
= walk_memory_regions_end(data
, pa
, prot
);
1808 for (i
= 0; i
< V_L2_SIZE
; ++i
) {
1809 pa
= base
| ((target_ulong
)i
<<
1810 (TARGET_PAGE_BITS
+ V_L2_BITS
* level
));
1811 rc
= walk_memory_regions_1(data
, pa
, level
- 1, pp
+ i
);
1821 int walk_memory_regions(void *priv
, walk_memory_regions_fn fn
)
1823 struct walk_memory_regions_data data
;
1831 for (i
= 0; i
< V_L1_SIZE
; i
++) {
1832 int rc
= walk_memory_regions_1(&data
, (target_ulong
)i
<< (V_L1_SHIFT
+ TARGET_PAGE_BITS
),
1833 V_L1_SHIFT
/ V_L2_BITS
- 1, l1_map
+ i
);
1839 return walk_memory_regions_end(&data
, 0, 0);
1842 static int dump_region(void *priv
, target_ulong start
,
1843 target_ulong end
, unsigned long prot
)
1845 FILE *f
= (FILE *)priv
;
1847 (void) fprintf(f
, TARGET_FMT_lx
"-"TARGET_FMT_lx
1848 " "TARGET_FMT_lx
" %c%c%c\n",
1849 start
, end
, end
- start
,
1850 ((prot
& PAGE_READ
) ? 'r' : '-'),
1851 ((prot
& PAGE_WRITE
) ? 'w' : '-'),
1852 ((prot
& PAGE_EXEC
) ? 'x' : '-'));
1857 /* dump memory mappings */
1858 void page_dump(FILE *f
)
1860 const int length
= sizeof(target_ulong
) * 2;
1861 (void) fprintf(f
, "%-*s %-*s %-*s %s\n",
1862 length
, "start", length
, "end", length
, "size", "prot");
1863 walk_memory_regions(f
, dump_region
);
1866 int page_get_flags(target_ulong address
)
1870 p
= page_find(address
>> TARGET_PAGE_BITS
);
1877 /* Modify the flags of a page and invalidate the code if necessary.
1878 The flag PAGE_WRITE_ORG is positioned automatically depending
1879 on PAGE_WRITE. The mmap_lock should already be held. */
1880 void page_set_flags(target_ulong start
, target_ulong end
, int flags
)
1882 target_ulong addr
, len
;
1884 /* This function should never be called with addresses outside the
1885 guest address space. If this assert fires, it probably indicates
1886 a missing call to h2g_valid. */
1887 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1888 assert(end
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1890 assert(start
< end
);
1892 start
= start
& TARGET_PAGE_MASK
;
1893 end
= TARGET_PAGE_ALIGN(end
);
1895 if (flags
& PAGE_WRITE
) {
1896 flags
|= PAGE_WRITE_ORG
;
1899 for (addr
= start
, len
= end
- start
;
1901 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1902 PageDesc
*p
= page_find_alloc(addr
>> TARGET_PAGE_BITS
, 1);
1904 /* If the write protection bit is set, then we invalidate
1906 if (!(p
->flags
& PAGE_WRITE
) &&
1907 (flags
& PAGE_WRITE
) &&
1909 tb_invalidate_phys_page(addr
, 0, NULL
, false);
1915 int page_check_range(target_ulong start
, target_ulong len
, int flags
)
1921 /* This function should never be called with addresses outside the
1922 guest address space. If this assert fires, it probably indicates
1923 a missing call to h2g_valid. */
1924 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1925 assert(start
< ((target_ulong
)1 << L1_MAP_ADDR_SPACE_BITS
));
1931 if (start
+ len
- 1 < start
) {
1932 /* We've wrapped around. */
1936 /* must do before we loose bits in the next step */
1937 end
= TARGET_PAGE_ALIGN(start
+ len
);
1938 start
= start
& TARGET_PAGE_MASK
;
1940 for (addr
= start
, len
= end
- start
;
1942 len
-= TARGET_PAGE_SIZE
, addr
+= TARGET_PAGE_SIZE
) {
1943 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1947 if (!(p
->flags
& PAGE_VALID
)) {
1951 if ((flags
& PAGE_READ
) && !(p
->flags
& PAGE_READ
)) {
1954 if (flags
& PAGE_WRITE
) {
1955 if (!(p
->flags
& PAGE_WRITE_ORG
)) {
1958 /* unprotect the page if it was put read-only because it
1959 contains translated code */
1960 if (!(p
->flags
& PAGE_WRITE
)) {
1961 if (!page_unprotect(addr
, 0, NULL
)) {
1970 /* called from signal handler: invalidate the code and unprotect the
1971 page. Return TRUE if the fault was successfully handled. */
1972 int page_unprotect(target_ulong address
, uintptr_t pc
, void *puc
)
1976 target_ulong host_start
, host_end
, addr
;
1978 /* Technically this isn't safe inside a signal handler. However we
1979 know this only ever happens in a synchronous SEGV handler, so in
1980 practice it seems to be ok. */
1983 p
= page_find(address
>> TARGET_PAGE_BITS
);
1989 /* if the page was really writable, then we change its
1990 protection back to writable */
1991 if ((p
->flags
& PAGE_WRITE_ORG
) && !(p
->flags
& PAGE_WRITE
)) {
1992 host_start
= address
& qemu_host_page_mask
;
1993 host_end
= host_start
+ qemu_host_page_size
;
1996 for (addr
= host_start
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
1997 p
= page_find(addr
>> TARGET_PAGE_BITS
);
1998 p
->flags
|= PAGE_WRITE
;
2001 /* and since the content will be modified, we must invalidate
2002 the corresponding translated code. */
2003 tb_invalidate_phys_page(addr
, pc
, puc
, true);
2004 #ifdef DEBUG_TB_CHECK
2005 tb_invalidate_check(addr
);
2008 mprotect((void *)g2h(host_start
), qemu_host_page_size
,
2017 #endif /* CONFIG_USER_ONLY */