cpu: move exec-all.h inclusion out of cpu.h
[qemu/ar7.git] / translate-all.c
blob1c1c85530af600f5cf08c1bb95f8d048aa7a58c3
1 /*
2 * Host code generation
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/mman.h>
23 #endif
24 #include "qemu/osdep.h"
27 #include "qemu-common.h"
28 #define NO_CPU_IO_DEFS
29 #include "cpu.h"
30 #include "trace.h"
31 #include "disas/disas.h"
32 #include "exec/exec-all.h"
33 #include "tcg.h"
34 #if defined(CONFIG_USER_ONLY)
35 #include "qemu.h"
36 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
37 #include <sys/param.h>
38 #if __FreeBSD_version >= 700104
39 #define HAVE_KINFO_GETVMMAP
40 #define sigqueue sigqueue_freebsd /* avoid redefinition */
41 #include <sys/proc.h>
42 #include <machine/profile.h>
43 #define _KERNEL
44 #include <sys/user.h>
45 #undef _KERNEL
46 #undef sigqueue
47 #include <libutil.h>
48 #endif
49 #endif
50 #else
51 #include "exec/address-spaces.h"
52 #endif
54 #include "exec/cputlb.h"
55 #include "exec/tb-hash.h"
56 #include "translate-all.h"
57 #include "qemu/bitmap.h"
58 #include "qemu/timer.h"
59 #include "exec/log.h"
61 //#define DEBUG_TB_INVALIDATE
62 //#define DEBUG_FLUSH
63 /* make various TB consistency checks */
64 //#define DEBUG_TB_CHECK
66 #if !defined(CONFIG_USER_ONLY)
67 /* TB consistency checks only implemented for usermode emulation. */
68 #undef DEBUG_TB_CHECK
69 #endif
71 #define SMC_BITMAP_USE_THRESHOLD 10
73 typedef struct PageDesc {
74 /* list of TBs intersecting this ram page */
75 TranslationBlock *first_tb;
76 #ifdef CONFIG_SOFTMMU
77 /* in order to optimize self modifying code, we count the number
78 of lookups we do to a given page to use a bitmap */
79 unsigned int code_write_count;
80 unsigned long *code_bitmap;
81 #else
82 unsigned long flags;
83 #endif
84 } PageDesc;
86 /* In system mode we want L1_MAP to be based on ram offsets,
87 while in user mode we want it to be based on virtual addresses. */
88 #if !defined(CONFIG_USER_ONLY)
89 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
90 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
91 #else
92 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
93 #endif
94 #else
95 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
96 #endif
98 /* Size of the L2 (and L3, etc) page tables. */
99 #define V_L2_BITS 10
100 #define V_L2_SIZE (1 << V_L2_BITS)
102 /* The bits remaining after N lower levels of page tables. */
103 #define V_L1_BITS_REM \
104 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
106 #if V_L1_BITS_REM < 4
107 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
108 #else
109 #define V_L1_BITS V_L1_BITS_REM
110 #endif
112 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
114 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
116 uintptr_t qemu_host_page_size;
117 intptr_t qemu_host_page_mask;
119 /* The bottom level has pointers to PageDesc */
120 static void *l1_map[V_L1_SIZE];
122 /* code generation context */
123 TCGContext tcg_ctx;
125 /* translation block context */
126 #ifdef CONFIG_USER_ONLY
127 __thread int have_tb_lock;
128 #endif
130 void tb_lock(void)
132 #ifdef CONFIG_USER_ONLY
133 assert(!have_tb_lock);
134 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
135 have_tb_lock++;
136 #endif
139 void tb_unlock(void)
141 #ifdef CONFIG_USER_ONLY
142 assert(have_tb_lock);
143 have_tb_lock--;
144 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
145 #endif
148 void tb_lock_reset(void)
150 #ifdef CONFIG_USER_ONLY
151 if (have_tb_lock) {
152 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
153 have_tb_lock = 0;
155 #endif
158 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
160 void cpu_gen_init(void)
162 tcg_context_init(&tcg_ctx);
165 /* Encode VAL as a signed leb128 sequence at P.
166 Return P incremented past the encoded value. */
167 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
169 int more, byte;
171 do {
172 byte = val & 0x7f;
173 val >>= 7;
174 more = !((val == 0 && (byte & 0x40) == 0)
175 || (val == -1 && (byte & 0x40) != 0));
176 if (more) {
177 byte |= 0x80;
179 *p++ = byte;
180 } while (more);
182 return p;
185 /* Decode a signed leb128 sequence at *PP; increment *PP past the
186 decoded value. Return the decoded value. */
187 static target_long decode_sleb128(uint8_t **pp)
189 uint8_t *p = *pp;
190 target_long val = 0;
191 int byte, shift = 0;
193 do {
194 byte = *p++;
195 val |= (target_ulong)(byte & 0x7f) << shift;
196 shift += 7;
197 } while (byte & 0x80);
198 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
199 val |= -(target_ulong)1 << shift;
202 *pp = p;
203 return val;
206 /* Encode the data collected about the instructions while compiling TB.
207 Place the data at BLOCK, and return the number of bytes consumed.
209 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
210 which come from the target's insn_start data, followed by a uintptr_t
211 which comes from the host pc of the end of the code implementing the insn.
213 Each line of the table is encoded as sleb128 deltas from the previous
214 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
215 That is, the first column is seeded with the guest pc, the last column
216 with the host pc, and the middle columns with zeros. */
218 static int encode_search(TranslationBlock *tb, uint8_t *block)
220 uint8_t *highwater = tcg_ctx.code_gen_highwater;
221 uint8_t *p = block;
222 int i, j, n;
224 tb->tc_search = block;
226 for (i = 0, n = tb->icount; i < n; ++i) {
227 target_ulong prev;
229 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
230 if (i == 0) {
231 prev = (j == 0 ? tb->pc : 0);
232 } else {
233 prev = tcg_ctx.gen_insn_data[i - 1][j];
235 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
237 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
238 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
240 /* Test for (pending) buffer overflow. The assumption is that any
241 one row beginning below the high water mark cannot overrun
242 the buffer completely. Thus we can test for overflow after
243 encoding a row without having to check during encoding. */
244 if (unlikely(p > highwater)) {
245 return -1;
249 return p - block;
252 /* The cpu state corresponding to 'searched_pc' is restored. */
253 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
254 uintptr_t searched_pc)
256 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
257 uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
258 CPUArchState *env = cpu->env_ptr;
259 uint8_t *p = tb->tc_search;
260 int i, j, num_insns = tb->icount;
261 #ifdef CONFIG_PROFILER
262 int64_t ti = profile_getclock();
263 #endif
265 if (searched_pc < host_pc) {
266 return -1;
269 /* Reconstruct the stored insn data while looking for the point at
270 which the end of the insn exceeds the searched_pc. */
271 for (i = 0; i < num_insns; ++i) {
272 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
273 data[j] += decode_sleb128(&p);
275 host_pc += decode_sleb128(&p);
276 if (host_pc > searched_pc) {
277 goto found;
280 return -1;
282 found:
283 if (tb->cflags & CF_USE_ICOUNT) {
284 assert(use_icount);
285 /* Reset the cycle counter to the start of the block. */
286 cpu->icount_decr.u16.low += num_insns;
287 /* Clear the IO flag. */
288 cpu->can_do_io = 0;
290 cpu->icount_decr.u16.low -= i;
291 restore_state_to_opc(env, tb, data);
293 #ifdef CONFIG_PROFILER
294 tcg_ctx.restore_time += profile_getclock() - ti;
295 tcg_ctx.restore_count++;
296 #endif
297 return 0;
300 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
302 TranslationBlock *tb;
304 tb = tb_find_pc(retaddr);
305 if (tb) {
306 cpu_restore_state_from_tb(cpu, tb, retaddr);
307 if (tb->cflags & CF_NOCACHE) {
308 /* one-shot translation, invalidate it immediately */
309 tb_phys_invalidate(tb, -1);
310 tb_free(tb);
312 return true;
314 return false;
317 void page_size_init(void)
319 /* NOTE: we can always suppose that qemu_host_page_size >=
320 TARGET_PAGE_SIZE */
321 qemu_real_host_page_size = getpagesize();
322 qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
323 if (qemu_host_page_size == 0) {
324 qemu_host_page_size = qemu_real_host_page_size;
326 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
327 qemu_host_page_size = TARGET_PAGE_SIZE;
329 qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
332 static void page_init(void)
334 page_size_init();
335 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
337 #ifdef HAVE_KINFO_GETVMMAP
338 struct kinfo_vmentry *freep;
339 int i, cnt;
341 freep = kinfo_getvmmap(getpid(), &cnt);
342 if (freep) {
343 mmap_lock();
344 for (i = 0; i < cnt; i++) {
345 unsigned long startaddr, endaddr;
347 startaddr = freep[i].kve_start;
348 endaddr = freep[i].kve_end;
349 if (h2g_valid(startaddr)) {
350 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
352 if (h2g_valid(endaddr)) {
353 endaddr = h2g(endaddr);
354 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
355 } else {
356 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
357 endaddr = ~0ul;
358 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
359 #endif
363 free(freep);
364 mmap_unlock();
366 #else
367 FILE *f;
369 last_brk = (unsigned long)sbrk(0);
371 f = fopen("/compat/linux/proc/self/maps", "r");
372 if (f) {
373 mmap_lock();
375 do {
376 unsigned long startaddr, endaddr;
377 int n;
379 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
381 if (n == 2 && h2g_valid(startaddr)) {
382 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
384 if (h2g_valid(endaddr)) {
385 endaddr = h2g(endaddr);
386 } else {
387 endaddr = ~0ul;
389 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
391 } while (!feof(f));
393 fclose(f);
394 mmap_unlock();
396 #endif
398 #endif
401 /* If alloc=1:
402 * Called with mmap_lock held for user-mode emulation.
404 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
406 PageDesc *pd;
407 void **lp;
408 int i;
410 /* Level 1. Always allocated. */
411 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
413 /* Level 2..N-1. */
414 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
415 void **p = atomic_rcu_read(lp);
417 if (p == NULL) {
418 if (!alloc) {
419 return NULL;
421 p = g_new0(void *, V_L2_SIZE);
422 atomic_rcu_set(lp, p);
425 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
428 pd = atomic_rcu_read(lp);
429 if (pd == NULL) {
430 if (!alloc) {
431 return NULL;
433 pd = g_new0(PageDesc, V_L2_SIZE);
434 atomic_rcu_set(lp, pd);
437 return pd + (index & (V_L2_SIZE - 1));
440 static inline PageDesc *page_find(tb_page_addr_t index)
442 return page_find_alloc(index, 0);
445 #if defined(CONFIG_USER_ONLY)
446 /* Currently it is not recommended to allocate big chunks of data in
447 user mode. It will change when a dedicated libc will be used. */
448 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
449 region in which the guest needs to run. Revisit this. */
450 #define USE_STATIC_CODE_GEN_BUFFER
451 #endif
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__powerpc64__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
466 #elif defined(__powerpc__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
468 #elif defined(__aarch64__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
470 #elif defined(__arm__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
472 #elif defined(__s390x__)
473 /* We have a +- 4GB range on the branches; leave some slop. */
474 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
475 #elif defined(__mips__)
476 /* We have a 256MB branch region, but leave room to make sure the
477 main executable is also within that region. */
478 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
479 #else
480 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
481 #endif
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
485 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
486 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
487 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
489 static inline size_t size_code_gen_buffer(size_t tb_size)
491 /* Size the buffer. */
492 if (tb_size == 0) {
493 #ifdef USE_STATIC_CODE_GEN_BUFFER
494 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
495 #else
496 /* ??? Needs adjustments. */
497 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
498 static buffer, we could size this on RESERVED_VA, on the text
499 segment size of the executable, or continue to use the default. */
500 tb_size = (unsigned long)(ram_size / 4);
501 #endif
503 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
504 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
506 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
507 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
509 return tb_size;
512 #ifdef __mips__
513 /* In order to use J and JAL within the code_gen_buffer, we require
514 that the buffer not cross a 256MB boundary. */
515 static inline bool cross_256mb(void *addr, size_t size)
517 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
520 /* We weren't able to allocate a buffer without crossing that boundary,
521 so make do with the larger portion of the buffer that doesn't cross.
522 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
523 static inline void *split_cross_256mb(void *buf1, size_t size1)
525 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
526 size_t size2 = buf1 + size1 - buf2;
528 size1 = buf2 - buf1;
529 if (size1 < size2) {
530 size1 = size2;
531 buf1 = buf2;
534 tcg_ctx.code_gen_buffer_size = size1;
535 return buf1;
537 #endif
539 #ifdef USE_STATIC_CODE_GEN_BUFFER
540 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
541 __attribute__((aligned(CODE_GEN_ALIGN)));
543 # ifdef _WIN32
544 static inline void do_protect(void *addr, long size, int prot)
546 DWORD old_protect;
547 VirtualProtect(addr, size, prot, &old_protect);
550 static inline void map_exec(void *addr, long size)
552 do_protect(addr, size, PAGE_EXECUTE_READWRITE);
555 static inline void map_none(void *addr, long size)
557 do_protect(addr, size, PAGE_NOACCESS);
559 # else
560 static inline void do_protect(void *addr, long size, int prot)
562 uintptr_t start, end;
564 start = (uintptr_t)addr;
565 start &= qemu_real_host_page_mask;
567 end = (uintptr_t)addr + size;
568 end = ROUND_UP(end, qemu_real_host_page_size);
570 mprotect((void *)start, end - start, prot);
573 static inline void map_exec(void *addr, long size)
575 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
578 static inline void map_none(void *addr, long size)
580 do_protect(addr, size, PROT_NONE);
582 # endif /* WIN32 */
584 static inline void *alloc_code_gen_buffer(void)
586 void *buf = static_code_gen_buffer;
587 size_t full_size, size;
589 /* The size of the buffer, rounded down to end on a page boundary. */
590 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
591 & qemu_real_host_page_mask) - (uintptr_t)buf;
593 /* Reserve a guard page. */
594 size = full_size - qemu_real_host_page_size;
596 /* Honor a command-line option limiting the size of the buffer. */
597 if (size > tcg_ctx.code_gen_buffer_size) {
598 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
599 & qemu_real_host_page_mask) - (uintptr_t)buf;
601 tcg_ctx.code_gen_buffer_size = size;
603 #ifdef __mips__
604 if (cross_256mb(buf, size)) {
605 buf = split_cross_256mb(buf, size);
606 size = tcg_ctx.code_gen_buffer_size;
608 #endif
610 map_exec(buf, size);
611 map_none(buf + size, qemu_real_host_page_size);
612 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
614 return buf;
616 #elif defined(_WIN32)
617 static inline void *alloc_code_gen_buffer(void)
619 size_t size = tcg_ctx.code_gen_buffer_size;
620 void *buf1, *buf2;
622 /* Perform the allocation in two steps, so that the guard page
623 is reserved but uncommitted. */
624 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
625 MEM_RESERVE, PAGE_NOACCESS);
626 if (buf1 != NULL) {
627 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
628 assert(buf1 == buf2);
631 return buf1;
633 #else
634 static inline void *alloc_code_gen_buffer(void)
636 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
637 uintptr_t start = 0;
638 size_t size = tcg_ctx.code_gen_buffer_size;
639 void *buf;
641 /* Constrain the position of the buffer based on the host cpu.
642 Note that these addresses are chosen in concert with the
643 addresses assigned in the relevant linker script file. */
644 # if defined(__PIE__) || defined(__PIC__)
645 /* Don't bother setting a preferred location if we're building
646 a position-independent executable. We're more likely to get
647 an address near the main executable if we let the kernel
648 choose the address. */
649 # elif defined(__x86_64__) && defined(MAP_32BIT)
650 /* Force the memory down into low memory with the executable.
651 Leave the choice of exact location with the kernel. */
652 flags |= MAP_32BIT;
653 /* Cannot expect to map more than 800MB in low memory. */
654 if (size > 800u * 1024 * 1024) {
655 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
657 # elif defined(__sparc__)
658 start = 0x40000000ul;
659 # elif defined(__s390x__)
660 start = 0x90000000ul;
661 # elif defined(__mips__)
662 # if _MIPS_SIM == _ABI64
663 start = 0x128000000ul;
664 # else
665 start = 0x08000000ul;
666 # endif
667 # endif
669 buf = mmap((void *)start, size + qemu_real_host_page_size,
670 PROT_NONE, flags, -1, 0);
671 if (buf == MAP_FAILED) {
672 return NULL;
675 #ifdef __mips__
676 if (cross_256mb(buf, size)) {
677 /* Try again, with the original still mapped, to avoid re-acquiring
678 that 256mb crossing. This time don't specify an address. */
679 size_t size2;
680 void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
681 PROT_NONE, flags, -1, 0);
682 switch (buf2 != MAP_FAILED) {
683 case 1:
684 if (!cross_256mb(buf2, size)) {
685 /* Success! Use the new buffer. */
686 munmap(buf, size + qemu_real_host_page_size);
687 break;
689 /* Failure. Work with what we had. */
690 munmap(buf2, size + qemu_real_host_page_size);
691 /* fallthru */
692 default:
693 /* Split the original buffer. Free the smaller half. */
694 buf2 = split_cross_256mb(buf, size);
695 size2 = tcg_ctx.code_gen_buffer_size;
696 if (buf == buf2) {
697 munmap(buf + size2 + qemu_real_host_page_size, size - size2);
698 } else {
699 munmap(buf, size - size2);
701 size = size2;
702 break;
704 buf = buf2;
706 #endif
708 /* Make the final buffer accessible. The guard page at the end
709 will remain inaccessible with PROT_NONE. */
710 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
712 /* Request large pages for the buffer. */
713 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
715 return buf;
717 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
719 static inline void code_gen_alloc(size_t tb_size)
721 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
722 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
723 if (tcg_ctx.code_gen_buffer == NULL) {
724 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
725 exit(1);
728 /* Estimate a good size for the number of TBs we can support. We
729 still haven't deducted the prologue from the buffer size here,
730 but that's minimal and won't affect the estimate much. */
731 tcg_ctx.code_gen_max_blocks
732 = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
733 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
735 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
738 /* Must be called before using the QEMU cpus. 'tb_size' is the size
739 (in bytes) allocated to the translation buffer. Zero means default
740 size. */
741 void tcg_exec_init(unsigned long tb_size)
743 cpu_gen_init();
744 page_init();
745 code_gen_alloc(tb_size);
746 #if defined(CONFIG_SOFTMMU)
747 /* There's no guest base to take into account, so go ahead and
748 initialize the prologue now. */
749 tcg_prologue_init(&tcg_ctx);
750 #endif
753 bool tcg_enabled(void)
755 return tcg_ctx.code_gen_buffer != NULL;
758 /* Allocate a new translation block. Flush the translation buffer if
759 too many translation blocks or too much generated code. */
760 static TranslationBlock *tb_alloc(target_ulong pc)
762 TranslationBlock *tb;
764 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
765 return NULL;
767 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
768 tb->pc = pc;
769 tb->cflags = 0;
770 return tb;
773 void tb_free(TranslationBlock *tb)
775 /* In practice this is mostly used for single use temporary TB
776 Ignore the hard cases and just back up if this TB happens to
777 be the last one generated. */
778 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
779 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
780 tcg_ctx.code_gen_ptr = tb->tc_ptr;
781 tcg_ctx.tb_ctx.nb_tbs--;
785 static inline void invalidate_page_bitmap(PageDesc *p)
787 #ifdef CONFIG_SOFTMMU
788 g_free(p->code_bitmap);
789 p->code_bitmap = NULL;
790 p->code_write_count = 0;
791 #endif
794 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
795 static void page_flush_tb_1(int level, void **lp)
797 int i;
799 if (*lp == NULL) {
800 return;
802 if (level == 0) {
803 PageDesc *pd = *lp;
805 for (i = 0; i < V_L2_SIZE; ++i) {
806 pd[i].first_tb = NULL;
807 invalidate_page_bitmap(pd + i);
809 } else {
810 void **pp = *lp;
812 for (i = 0; i < V_L2_SIZE; ++i) {
813 page_flush_tb_1(level - 1, pp + i);
818 static void page_flush_tb(void)
820 int i;
822 for (i = 0; i < V_L1_SIZE; i++) {
823 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
827 /* flush all the translation blocks */
828 /* XXX: tb_flush is currently not thread safe */
829 void tb_flush(CPUState *cpu)
831 #if defined(DEBUG_FLUSH)
832 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
833 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
834 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
835 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
836 tcg_ctx.tb_ctx.nb_tbs : 0);
837 #endif
838 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
839 > tcg_ctx.code_gen_buffer_size) {
840 cpu_abort(cpu, "Internal error: code buffer overflow\n");
842 tcg_ctx.tb_ctx.nb_tbs = 0;
844 CPU_FOREACH(cpu) {
845 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
846 cpu->tb_flushed = true;
849 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
850 page_flush_tb();
852 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
853 /* XXX: flush processor icache at this point if cache flush is
854 expensive */
855 tcg_ctx.tb_ctx.tb_flush_count++;
858 #ifdef DEBUG_TB_CHECK
860 static void tb_invalidate_check(target_ulong address)
862 TranslationBlock *tb;
863 int i;
865 address &= TARGET_PAGE_MASK;
866 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
867 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
868 tb = tb->phys_hash_next) {
869 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
870 address >= tb->pc + tb->size)) {
871 printf("ERROR invalidate: address=" TARGET_FMT_lx
872 " PC=%08lx size=%04x\n",
873 address, (long)tb->pc, tb->size);
879 /* verify that all the pages have correct rights for code */
880 static void tb_page_check(void)
882 TranslationBlock *tb;
883 int i, flags1, flags2;
885 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
886 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
887 tb = tb->phys_hash_next) {
888 flags1 = page_get_flags(tb->pc);
889 flags2 = page_get_flags(tb->pc + tb->size - 1);
890 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
891 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
892 (long)tb->pc, tb->size, flags1, flags2);
898 #endif
900 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
902 TranslationBlock *tb1;
904 for (;;) {
905 tb1 = *ptb;
906 if (tb1 == tb) {
907 *ptb = tb1->phys_hash_next;
908 break;
910 ptb = &tb1->phys_hash_next;
914 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
916 TranslationBlock *tb1;
917 unsigned int n1;
919 for (;;) {
920 tb1 = *ptb;
921 n1 = (uintptr_t)tb1 & 3;
922 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
923 if (tb1 == tb) {
924 *ptb = tb1->page_next[n1];
925 break;
927 ptb = &tb1->page_next[n1];
931 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
932 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
934 TranslationBlock *tb1;
935 uintptr_t *ptb, ntb;
936 unsigned int n1;
938 ptb = &tb->jmp_list_next[n];
939 if (*ptb) {
940 /* find tb(n) in circular list */
941 for (;;) {
942 ntb = *ptb;
943 n1 = ntb & 3;
944 tb1 = (TranslationBlock *)(ntb & ~3);
945 if (n1 == n && tb1 == tb) {
946 break;
948 if (n1 == 2) {
949 ptb = &tb1->jmp_list_first;
950 } else {
951 ptb = &tb1->jmp_list_next[n1];
954 /* now we can suppress tb(n) from the list */
955 *ptb = tb->jmp_list_next[n];
957 tb->jmp_list_next[n] = (uintptr_t)NULL;
961 /* reset the jump entry 'n' of a TB so that it is not chained to
962 another TB */
963 static inline void tb_reset_jump(TranslationBlock *tb, int n)
965 uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
966 tb_set_jmp_target(tb, n, addr);
969 /* remove any jumps to the TB */
970 static inline void tb_jmp_unlink(TranslationBlock *tb)
972 TranslationBlock *tb1;
973 uintptr_t *ptb, ntb;
974 unsigned int n1;
976 ptb = &tb->jmp_list_first;
977 for (;;) {
978 ntb = *ptb;
979 n1 = ntb & 3;
980 tb1 = (TranslationBlock *)(ntb & ~3);
981 if (n1 == 2) {
982 break;
984 tb_reset_jump(tb1, n1);
985 *ptb = tb1->jmp_list_next[n1];
986 tb1->jmp_list_next[n1] = (uintptr_t)NULL;
990 /* invalidate one TB */
991 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
993 CPUState *cpu;
994 PageDesc *p;
995 unsigned int h;
996 tb_page_addr_t phys_pc;
998 /* remove the TB from the hash list */
999 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1000 h = tb_phys_hash_func(phys_pc);
1001 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
1003 /* remove the TB from the page list */
1004 if (tb->page_addr[0] != page_addr) {
1005 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1006 tb_page_remove(&p->first_tb, tb);
1007 invalidate_page_bitmap(p);
1009 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
1010 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1011 tb_page_remove(&p->first_tb, tb);
1012 invalidate_page_bitmap(p);
1015 /* remove the TB from the hash list */
1016 h = tb_jmp_cache_hash_func(tb->pc);
1017 CPU_FOREACH(cpu) {
1018 if (cpu->tb_jmp_cache[h] == tb) {
1019 cpu->tb_jmp_cache[h] = NULL;
1023 /* suppress this TB from the two jump lists */
1024 tb_remove_from_jmp_list(tb, 0);
1025 tb_remove_from_jmp_list(tb, 1);
1027 /* suppress any remaining jumps to this TB */
1028 tb_jmp_unlink(tb);
1030 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
1033 #ifdef CONFIG_SOFTMMU
1034 static void build_page_bitmap(PageDesc *p)
1036 int n, tb_start, tb_end;
1037 TranslationBlock *tb;
1039 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1041 tb = p->first_tb;
1042 while (tb != NULL) {
1043 n = (uintptr_t)tb & 3;
1044 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1045 /* NOTE: this is subtle as a TB may span two physical pages */
1046 if (n == 0) {
1047 /* NOTE: tb_end may be after the end of the page, but
1048 it is not a problem */
1049 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1050 tb_end = tb_start + tb->size;
1051 if (tb_end > TARGET_PAGE_SIZE) {
1052 tb_end = TARGET_PAGE_SIZE;
1054 } else {
1055 tb_start = 0;
1056 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1058 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1059 tb = tb->page_next[n];
1062 #endif
1064 /* add the tb in the target page and protect it if necessary
1066 * Called with mmap_lock held for user-mode emulation.
1068 static inline void tb_alloc_page(TranslationBlock *tb,
1069 unsigned int n, tb_page_addr_t page_addr)
1071 PageDesc *p;
1072 #ifndef CONFIG_USER_ONLY
1073 bool page_already_protected;
1074 #endif
1076 tb->page_addr[n] = page_addr;
1077 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1078 tb->page_next[n] = p->first_tb;
1079 #ifndef CONFIG_USER_ONLY
1080 page_already_protected = p->first_tb != NULL;
1081 #endif
1082 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1083 invalidate_page_bitmap(p);
1085 #if defined(CONFIG_USER_ONLY)
1086 if (p->flags & PAGE_WRITE) {
1087 target_ulong addr;
1088 PageDesc *p2;
1089 int prot;
1091 /* force the host page as non writable (writes will have a
1092 page fault + mprotect overhead) */
1093 page_addr &= qemu_host_page_mask;
1094 prot = 0;
1095 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1096 addr += TARGET_PAGE_SIZE) {
1098 p2 = page_find(addr >> TARGET_PAGE_BITS);
1099 if (!p2) {
1100 continue;
1102 prot |= p2->flags;
1103 p2->flags &= ~PAGE_WRITE;
1105 mprotect(g2h(page_addr), qemu_host_page_size,
1106 (prot & PAGE_BITS) & ~PAGE_WRITE);
1107 #ifdef DEBUG_TB_INVALIDATE
1108 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1109 page_addr);
1110 #endif
1112 #else
1113 /* if some code is already present, then the pages are already
1114 protected. So we handle the case where only the first TB is
1115 allocated in a physical page */
1116 if (!page_already_protected) {
1117 tlb_protect_code(page_addr);
1119 #endif
1122 /* add a new TB and link it to the physical page tables. phys_page2 is
1123 * (-1) to indicate that only one page contains the TB.
1125 * Called with mmap_lock held for user-mode emulation.
1127 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1128 tb_page_addr_t phys_page2)
1130 unsigned int h;
1131 TranslationBlock **ptb;
1133 /* add in the physical hash table */
1134 h = tb_phys_hash_func(phys_pc);
1135 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1136 tb->phys_hash_next = *ptb;
1137 *ptb = tb;
1139 /* add in the page list */
1140 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1141 if (phys_page2 != -1) {
1142 tb_alloc_page(tb, 1, phys_page2);
1143 } else {
1144 tb->page_addr[1] = -1;
1147 #ifdef DEBUG_TB_CHECK
1148 tb_page_check();
1149 #endif
1152 /* Called with mmap_lock held for user mode emulation. */
1153 TranslationBlock *tb_gen_code(CPUState *cpu,
1154 target_ulong pc, target_ulong cs_base,
1155 uint32_t flags, int cflags)
1157 CPUArchState *env = cpu->env_ptr;
1158 TranslationBlock *tb;
1159 tb_page_addr_t phys_pc, phys_page2;
1160 target_ulong virt_page2;
1161 tcg_insn_unit *gen_code_buf;
1162 int gen_code_size, search_size;
1163 #ifdef CONFIG_PROFILER
1164 int64_t ti;
1165 #endif
1167 phys_pc = get_page_addr_code(env, pc);
1168 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
1169 cflags |= CF_USE_ICOUNT;
1172 tb = tb_alloc(pc);
1173 if (unlikely(!tb)) {
1174 buffer_overflow:
1175 /* flush must be done */
1176 tb_flush(cpu);
1177 /* cannot fail at this point */
1178 tb = tb_alloc(pc);
1179 assert(tb != NULL);
1182 gen_code_buf = tcg_ctx.code_gen_ptr;
1183 tb->tc_ptr = gen_code_buf;
1184 tb->cs_base = cs_base;
1185 tb->flags = flags;
1186 tb->cflags = cflags;
1188 #ifdef CONFIG_PROFILER
1189 tcg_ctx.tb_count1++; /* includes aborted translations because of
1190 exceptions */
1191 ti = profile_getclock();
1192 #endif
1194 tcg_func_start(&tcg_ctx);
1196 gen_intermediate_code(env, tb);
1198 trace_translate_block(tb, tb->pc, tb->tc_ptr);
1200 /* generate machine code */
1201 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1202 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1203 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
1204 #ifdef USE_DIRECT_JUMP
1205 tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
1206 tcg_ctx.tb_jmp_target_addr = NULL;
1207 #else
1208 tcg_ctx.tb_jmp_insn_offset = NULL;
1209 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;
1210 #endif
1212 #ifdef CONFIG_PROFILER
1213 tcg_ctx.tb_count++;
1214 tcg_ctx.interm_time += profile_getclock() - ti;
1215 tcg_ctx.code_time -= profile_getclock();
1216 #endif
1218 /* ??? Overflow could be handled better here. In particular, we
1219 don't need to re-do gen_intermediate_code, nor should we re-do
1220 the tcg optimization currently hidden inside tcg_gen_code. All
1221 that should be required is to flush the TBs, allocate a new TB,
1222 re-initialize it per above, and re-do the actual code generation. */
1223 gen_code_size = tcg_gen_code(&tcg_ctx, tb);
1224 if (unlikely(gen_code_size < 0)) {
1225 goto buffer_overflow;
1227 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1228 if (unlikely(search_size < 0)) {
1229 goto buffer_overflow;
1232 #ifdef CONFIG_PROFILER
1233 tcg_ctx.code_time += profile_getclock();
1234 tcg_ctx.code_in_len += tb->size;
1235 tcg_ctx.code_out_len += gen_code_size;
1236 tcg_ctx.search_out_len += search_size;
1237 #endif
1239 #ifdef DEBUG_DISAS
1240 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1241 qemu_log_in_addr_range(tb->pc)) {
1242 qemu_log("OUT: [size=%d]\n", gen_code_size);
1243 log_disas(tb->tc_ptr, gen_code_size);
1244 qemu_log("\n");
1245 qemu_log_flush();
1247 #endif
1249 tcg_ctx.code_gen_ptr = (void *)
1250 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1251 CODE_GEN_ALIGN);
1253 /* init jump list */
1254 assert(((uintptr_t)tb & 3) == 0);
1255 tb->jmp_list_first = (uintptr_t)tb | 2;
1256 tb->jmp_list_next[0] = (uintptr_t)NULL;
1257 tb->jmp_list_next[1] = (uintptr_t)NULL;
1259 /* init original jump addresses wich has been set during tcg_gen_code() */
1260 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1261 tb_reset_jump(tb, 0);
1263 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1264 tb_reset_jump(tb, 1);
1267 /* check next page if needed */
1268 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1269 phys_page2 = -1;
1270 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1271 phys_page2 = get_page_addr_code(env, virt_page2);
1273 /* As long as consistency of the TB stuff is provided by tb_lock in user
1274 * mode and is implicit in single-threaded softmmu emulation, no explicit
1275 * memory barrier is required before tb_link_page() makes the TB visible
1276 * through the physical hash table and physical page list.
1278 tb_link_page(tb, phys_pc, phys_page2);
1279 return tb;
1283 * Invalidate all TBs which intersect with the target physical address range
1284 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1285 * 'is_cpu_write_access' should be true if called from a real cpu write
1286 * access: the virtual CPU will exit the current TB if code is modified inside
1287 * this TB.
1289 * Called with mmap_lock held for user-mode emulation
1291 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1293 while (start < end) {
1294 tb_invalidate_phys_page_range(start, end, 0);
1295 start &= TARGET_PAGE_MASK;
1296 start += TARGET_PAGE_SIZE;
1301 * Invalidate all TBs which intersect with the target physical address range
1302 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1303 * 'is_cpu_write_access' should be true if called from a real cpu write
1304 * access: the virtual CPU will exit the current TB if code is modified inside
1305 * this TB.
1307 * Called with mmap_lock held for user-mode emulation
1309 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1310 int is_cpu_write_access)
1312 TranslationBlock *tb, *tb_next;
1313 #if defined(TARGET_HAS_PRECISE_SMC)
1314 CPUState *cpu = current_cpu;
1315 CPUArchState *env = NULL;
1316 #endif
1317 tb_page_addr_t tb_start, tb_end;
1318 PageDesc *p;
1319 int n;
1320 #ifdef TARGET_HAS_PRECISE_SMC
1321 int current_tb_not_found = is_cpu_write_access;
1322 TranslationBlock *current_tb = NULL;
1323 int current_tb_modified = 0;
1324 target_ulong current_pc = 0;
1325 target_ulong current_cs_base = 0;
1326 uint32_t current_flags = 0;
1327 #endif /* TARGET_HAS_PRECISE_SMC */
1329 p = page_find(start >> TARGET_PAGE_BITS);
1330 if (!p) {
1331 return;
1333 #if defined(TARGET_HAS_PRECISE_SMC)
1334 if (cpu != NULL) {
1335 env = cpu->env_ptr;
1337 #endif
1339 /* we remove all the TBs in the range [start, end[ */
1340 /* XXX: see if in some cases it could be faster to invalidate all
1341 the code */
1342 tb = p->first_tb;
1343 while (tb != NULL) {
1344 n = (uintptr_t)tb & 3;
1345 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1346 tb_next = tb->page_next[n];
1347 /* NOTE: this is subtle as a TB may span two physical pages */
1348 if (n == 0) {
1349 /* NOTE: tb_end may be after the end of the page, but
1350 it is not a problem */
1351 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1352 tb_end = tb_start + tb->size;
1353 } else {
1354 tb_start = tb->page_addr[1];
1355 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1357 if (!(tb_end <= start || tb_start >= end)) {
1358 #ifdef TARGET_HAS_PRECISE_SMC
1359 if (current_tb_not_found) {
1360 current_tb_not_found = 0;
1361 current_tb = NULL;
1362 if (cpu->mem_io_pc) {
1363 /* now we have a real cpu fault */
1364 current_tb = tb_find_pc(cpu->mem_io_pc);
1367 if (current_tb == tb &&
1368 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1369 /* If we are modifying the current TB, we must stop
1370 its execution. We could be more precise by checking
1371 that the modification is after the current PC, but it
1372 would require a specialized function to partially
1373 restore the CPU state */
1375 current_tb_modified = 1;
1376 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1377 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1378 &current_flags);
1380 #endif /* TARGET_HAS_PRECISE_SMC */
1381 tb_phys_invalidate(tb, -1);
1383 tb = tb_next;
1385 #if !defined(CONFIG_USER_ONLY)
1386 /* if no code remaining, no need to continue to use slow writes */
1387 if (!p->first_tb) {
1388 invalidate_page_bitmap(p);
1389 tlb_unprotect_code(start);
1391 #endif
1392 #ifdef TARGET_HAS_PRECISE_SMC
1393 if (current_tb_modified) {
1394 /* we generate a block containing just the instruction
1395 modifying the memory. It will ensure that it cannot modify
1396 itself */
1397 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1398 cpu_resume_from_signal(cpu, NULL);
1400 #endif
1403 #ifdef CONFIG_SOFTMMU
1404 /* len must be <= 8 and start must be a multiple of len */
1405 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1407 PageDesc *p;
1409 #if 0
1410 if (1) {
1411 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1412 cpu_single_env->mem_io_vaddr, len,
1413 cpu_single_env->eip,
1414 cpu_single_env->eip +
1415 (intptr_t)cpu_single_env->segs[R_CS].base);
1417 #endif
1418 p = page_find(start >> TARGET_PAGE_BITS);
1419 if (!p) {
1420 return;
1422 if (!p->code_bitmap &&
1423 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1424 /* build code bitmap */
1425 build_page_bitmap(p);
1427 if (p->code_bitmap) {
1428 unsigned int nr;
1429 unsigned long b;
1431 nr = start & ~TARGET_PAGE_MASK;
1432 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1433 if (b & ((1 << len) - 1)) {
1434 goto do_invalidate;
1436 } else {
1437 do_invalidate:
1438 tb_invalidate_phys_page_range(start, start + len, 1);
1441 #else
1442 /* Called with mmap_lock held. */
1443 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1444 uintptr_t pc, void *puc,
1445 bool locked)
1447 TranslationBlock *tb;
1448 PageDesc *p;
1449 int n;
1450 #ifdef TARGET_HAS_PRECISE_SMC
1451 TranslationBlock *current_tb = NULL;
1452 CPUState *cpu = current_cpu;
1453 CPUArchState *env = NULL;
1454 int current_tb_modified = 0;
1455 target_ulong current_pc = 0;
1456 target_ulong current_cs_base = 0;
1457 uint32_t current_flags = 0;
1458 #endif
1460 addr &= TARGET_PAGE_MASK;
1461 p = page_find(addr >> TARGET_PAGE_BITS);
1462 if (!p) {
1463 return;
1465 tb = p->first_tb;
1466 #ifdef TARGET_HAS_PRECISE_SMC
1467 if (tb && pc != 0) {
1468 current_tb = tb_find_pc(pc);
1470 if (cpu != NULL) {
1471 env = cpu->env_ptr;
1473 #endif
1474 while (tb != NULL) {
1475 n = (uintptr_t)tb & 3;
1476 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1477 #ifdef TARGET_HAS_PRECISE_SMC
1478 if (current_tb == tb &&
1479 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1480 /* If we are modifying the current TB, we must stop
1481 its execution. We could be more precise by checking
1482 that the modification is after the current PC, but it
1483 would require a specialized function to partially
1484 restore the CPU state */
1486 current_tb_modified = 1;
1487 cpu_restore_state_from_tb(cpu, current_tb, pc);
1488 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1489 &current_flags);
1491 #endif /* TARGET_HAS_PRECISE_SMC */
1492 tb_phys_invalidate(tb, addr);
1493 tb = tb->page_next[n];
1495 p->first_tb = NULL;
1496 #ifdef TARGET_HAS_PRECISE_SMC
1497 if (current_tb_modified) {
1498 /* we generate a block containing just the instruction
1499 modifying the memory. It will ensure that it cannot modify
1500 itself */
1501 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1502 if (locked) {
1503 mmap_unlock();
1505 cpu_resume_from_signal(cpu, puc);
1507 #endif
1509 #endif
1511 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1512 tb[1].tc_ptr. Return NULL if not found */
1513 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1515 int m_min, m_max, m;
1516 uintptr_t v;
1517 TranslationBlock *tb;
1519 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1520 return NULL;
1522 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1523 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1524 return NULL;
1526 /* binary search (cf Knuth) */
1527 m_min = 0;
1528 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1529 while (m_min <= m_max) {
1530 m = (m_min + m_max) >> 1;
1531 tb = &tcg_ctx.tb_ctx.tbs[m];
1532 v = (uintptr_t)tb->tc_ptr;
1533 if (v == tc_ptr) {
1534 return tb;
1535 } else if (tc_ptr < v) {
1536 m_max = m - 1;
1537 } else {
1538 m_min = m + 1;
1541 return &tcg_ctx.tb_ctx.tbs[m_max];
1544 #if !defined(CONFIG_USER_ONLY)
1545 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1547 ram_addr_t ram_addr;
1548 MemoryRegion *mr;
1549 hwaddr l = 1;
1551 rcu_read_lock();
1552 mr = address_space_translate(as, addr, &addr, &l, false);
1553 if (!(memory_region_is_ram(mr)
1554 || memory_region_is_romd(mr))) {
1555 rcu_read_unlock();
1556 return;
1558 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1559 + addr;
1560 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1561 rcu_read_unlock();
1563 #endif /* !defined(CONFIG_USER_ONLY) */
1565 void tb_check_watchpoint(CPUState *cpu)
1567 TranslationBlock *tb;
1569 tb = tb_find_pc(cpu->mem_io_pc);
1570 if (tb) {
1571 /* We can use retranslation to find the PC. */
1572 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1573 tb_phys_invalidate(tb, -1);
1574 } else {
1575 /* The exception probably happened in a helper. The CPU state should
1576 have been saved before calling it. Fetch the PC from there. */
1577 CPUArchState *env = cpu->env_ptr;
1578 target_ulong pc, cs_base;
1579 tb_page_addr_t addr;
1580 uint32_t flags;
1582 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1583 addr = get_page_addr_code(env, pc);
1584 tb_invalidate_phys_range(addr, addr + 1);
1588 #ifndef CONFIG_USER_ONLY
1589 /* in deterministic execution mode, instructions doing device I/Os
1590 must be at the end of the TB */
1591 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1593 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1594 CPUArchState *env = cpu->env_ptr;
1595 #endif
1596 TranslationBlock *tb;
1597 uint32_t n, cflags;
1598 target_ulong pc, cs_base;
1599 uint32_t flags;
1601 tb = tb_find_pc(retaddr);
1602 if (!tb) {
1603 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1604 (void *)retaddr);
1606 n = cpu->icount_decr.u16.low + tb->icount;
1607 cpu_restore_state_from_tb(cpu, tb, retaddr);
1608 /* Calculate how many instructions had been executed before the fault
1609 occurred. */
1610 n = n - cpu->icount_decr.u16.low;
1611 /* Generate a new TB ending on the I/O insn. */
1612 n++;
1613 /* On MIPS and SH, delay slot instructions can only be restarted if
1614 they were already the first instruction in the TB. If this is not
1615 the first instruction in a TB then re-execute the preceding
1616 branch. */
1617 #if defined(TARGET_MIPS)
1618 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1619 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1620 cpu->icount_decr.u16.low++;
1621 env->hflags &= ~MIPS_HFLAG_BMASK;
1623 #elif defined(TARGET_SH4)
1624 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1625 && n > 1) {
1626 env->pc -= 2;
1627 cpu->icount_decr.u16.low++;
1628 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1630 #endif
1631 /* This should never happen. */
1632 if (n > CF_COUNT_MASK) {
1633 cpu_abort(cpu, "TB too big during recompile");
1636 cflags = n | CF_LAST_IO;
1637 pc = tb->pc;
1638 cs_base = tb->cs_base;
1639 flags = tb->flags;
1640 tb_phys_invalidate(tb, -1);
1641 if (tb->cflags & CF_NOCACHE) {
1642 if (tb->orig_tb) {
1643 /* Invalidate original TB if this TB was generated in
1644 * cpu_exec_nocache() */
1645 tb_phys_invalidate(tb->orig_tb, -1);
1647 tb_free(tb);
1649 /* FIXME: In theory this could raise an exception. In practice
1650 we have already translated the block once so it's probably ok. */
1651 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1652 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1653 the first in the TB) then we end up generating a whole new TB and
1654 repeating the fault, which is horribly inefficient.
1655 Better would be to execute just this insn uncached, or generate a
1656 second new TB. */
1657 cpu_resume_from_signal(cpu, NULL);
1660 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1662 unsigned int i;
1664 /* Discard jump cache entries for any tb which might potentially
1665 overlap the flushed page. */
1666 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1667 memset(&cpu->tb_jmp_cache[i], 0,
1668 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1670 i = tb_jmp_cache_hash_page(addr);
1671 memset(&cpu->tb_jmp_cache[i], 0,
1672 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1675 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1677 int i, target_code_size, max_target_code_size;
1678 int direct_jmp_count, direct_jmp2_count, cross_page;
1679 TranslationBlock *tb;
1681 target_code_size = 0;
1682 max_target_code_size = 0;
1683 cross_page = 0;
1684 direct_jmp_count = 0;
1685 direct_jmp2_count = 0;
1686 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1687 tb = &tcg_ctx.tb_ctx.tbs[i];
1688 target_code_size += tb->size;
1689 if (tb->size > max_target_code_size) {
1690 max_target_code_size = tb->size;
1692 if (tb->page_addr[1] != -1) {
1693 cross_page++;
1695 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1696 direct_jmp_count++;
1697 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1698 direct_jmp2_count++;
1702 /* XXX: avoid using doubles ? */
1703 cpu_fprintf(f, "Translation buffer state:\n");
1704 cpu_fprintf(f, "gen code size %td/%zd\n",
1705 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1706 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
1707 cpu_fprintf(f, "TB count %d/%d\n",
1708 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1709 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1710 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1711 tcg_ctx.tb_ctx.nb_tbs : 0,
1712 max_target_code_size);
1713 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1714 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1715 tcg_ctx.code_gen_buffer) /
1716 tcg_ctx.tb_ctx.nb_tbs : 0,
1717 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1718 tcg_ctx.code_gen_buffer) /
1719 target_code_size : 0);
1720 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1721 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1722 tcg_ctx.tb_ctx.nb_tbs : 0);
1723 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1724 direct_jmp_count,
1725 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1726 tcg_ctx.tb_ctx.nb_tbs : 0,
1727 direct_jmp2_count,
1728 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1729 tcg_ctx.tb_ctx.nb_tbs : 0);
1730 cpu_fprintf(f, "\nStatistics:\n");
1731 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1732 cpu_fprintf(f, "TB invalidate count %d\n",
1733 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1734 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1735 tcg_dump_info(f, cpu_fprintf);
1738 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1740 tcg_dump_op_count(f, cpu_fprintf);
1743 #else /* CONFIG_USER_ONLY */
1745 void cpu_interrupt(CPUState *cpu, int mask)
1747 cpu->interrupt_request |= mask;
1748 cpu->tcg_exit_req = 1;
1752 * Walks guest process memory "regions" one by one
1753 * and calls callback function 'fn' for each region.
1755 struct walk_memory_regions_data {
1756 walk_memory_regions_fn fn;
1757 void *priv;
1758 target_ulong start;
1759 int prot;
1762 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1763 target_ulong end, int new_prot)
1765 if (data->start != -1u) {
1766 int rc = data->fn(data->priv, data->start, end, data->prot);
1767 if (rc != 0) {
1768 return rc;
1772 data->start = (new_prot ? end : -1u);
1773 data->prot = new_prot;
1775 return 0;
1778 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1779 target_ulong base, int level, void **lp)
1781 target_ulong pa;
1782 int i, rc;
1784 if (*lp == NULL) {
1785 return walk_memory_regions_end(data, base, 0);
1788 if (level == 0) {
1789 PageDesc *pd = *lp;
1791 for (i = 0; i < V_L2_SIZE; ++i) {
1792 int prot = pd[i].flags;
1794 pa = base | (i << TARGET_PAGE_BITS);
1795 if (prot != data->prot) {
1796 rc = walk_memory_regions_end(data, pa, prot);
1797 if (rc != 0) {
1798 return rc;
1802 } else {
1803 void **pp = *lp;
1805 for (i = 0; i < V_L2_SIZE; ++i) {
1806 pa = base | ((target_ulong)i <<
1807 (TARGET_PAGE_BITS + V_L2_BITS * level));
1808 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1809 if (rc != 0) {
1810 return rc;
1815 return 0;
1818 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1820 struct walk_memory_regions_data data;
1821 uintptr_t i;
1823 data.fn = fn;
1824 data.priv = priv;
1825 data.start = -1u;
1826 data.prot = 0;
1828 for (i = 0; i < V_L1_SIZE; i++) {
1829 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
1830 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1831 if (rc != 0) {
1832 return rc;
1836 return walk_memory_regions_end(&data, 0, 0);
1839 static int dump_region(void *priv, target_ulong start,
1840 target_ulong end, unsigned long prot)
1842 FILE *f = (FILE *)priv;
1844 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1845 " "TARGET_FMT_lx" %c%c%c\n",
1846 start, end, end - start,
1847 ((prot & PAGE_READ) ? 'r' : '-'),
1848 ((prot & PAGE_WRITE) ? 'w' : '-'),
1849 ((prot & PAGE_EXEC) ? 'x' : '-'));
1851 return 0;
1854 /* dump memory mappings */
1855 void page_dump(FILE *f)
1857 const int length = sizeof(target_ulong) * 2;
1858 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1859 length, "start", length, "end", length, "size", "prot");
1860 walk_memory_regions(f, dump_region);
1863 int page_get_flags(target_ulong address)
1865 PageDesc *p;
1867 p = page_find(address >> TARGET_PAGE_BITS);
1868 if (!p) {
1869 return 0;
1871 return p->flags;
1874 /* Modify the flags of a page and invalidate the code if necessary.
1875 The flag PAGE_WRITE_ORG is positioned automatically depending
1876 on PAGE_WRITE. The mmap_lock should already be held. */
1877 void page_set_flags(target_ulong start, target_ulong end, int flags)
1879 target_ulong addr, len;
1881 /* This function should never be called with addresses outside the
1882 guest address space. If this assert fires, it probably indicates
1883 a missing call to h2g_valid. */
1884 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1885 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1886 #endif
1887 assert(start < end);
1889 start = start & TARGET_PAGE_MASK;
1890 end = TARGET_PAGE_ALIGN(end);
1892 if (flags & PAGE_WRITE) {
1893 flags |= PAGE_WRITE_ORG;
1896 for (addr = start, len = end - start;
1897 len != 0;
1898 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1899 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1901 /* If the write protection bit is set, then we invalidate
1902 the code inside. */
1903 if (!(p->flags & PAGE_WRITE) &&
1904 (flags & PAGE_WRITE) &&
1905 p->first_tb) {
1906 tb_invalidate_phys_page(addr, 0, NULL, false);
1908 p->flags = flags;
1912 int page_check_range(target_ulong start, target_ulong len, int flags)
1914 PageDesc *p;
1915 target_ulong end;
1916 target_ulong addr;
1918 /* This function should never be called with addresses outside the
1919 guest address space. If this assert fires, it probably indicates
1920 a missing call to h2g_valid. */
1921 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1922 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1923 #endif
1925 if (len == 0) {
1926 return 0;
1928 if (start + len - 1 < start) {
1929 /* We've wrapped around. */
1930 return -1;
1933 /* must do before we loose bits in the next step */
1934 end = TARGET_PAGE_ALIGN(start + len);
1935 start = start & TARGET_PAGE_MASK;
1937 for (addr = start, len = end - start;
1938 len != 0;
1939 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1940 p = page_find(addr >> TARGET_PAGE_BITS);
1941 if (!p) {
1942 return -1;
1944 if (!(p->flags & PAGE_VALID)) {
1945 return -1;
1948 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1949 return -1;
1951 if (flags & PAGE_WRITE) {
1952 if (!(p->flags & PAGE_WRITE_ORG)) {
1953 return -1;
1955 /* unprotect the page if it was put read-only because it
1956 contains translated code */
1957 if (!(p->flags & PAGE_WRITE)) {
1958 if (!page_unprotect(addr, 0, NULL)) {
1959 return -1;
1964 return 0;
1967 /* called from signal handler: invalidate the code and unprotect the
1968 page. Return TRUE if the fault was successfully handled. */
1969 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1971 unsigned int prot;
1972 PageDesc *p;
1973 target_ulong host_start, host_end, addr;
1975 /* Technically this isn't safe inside a signal handler. However we
1976 know this only ever happens in a synchronous SEGV handler, so in
1977 practice it seems to be ok. */
1978 mmap_lock();
1980 p = page_find(address >> TARGET_PAGE_BITS);
1981 if (!p) {
1982 mmap_unlock();
1983 return 0;
1986 /* if the page was really writable, then we change its
1987 protection back to writable */
1988 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1989 host_start = address & qemu_host_page_mask;
1990 host_end = host_start + qemu_host_page_size;
1992 prot = 0;
1993 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1994 p = page_find(addr >> TARGET_PAGE_BITS);
1995 p->flags |= PAGE_WRITE;
1996 prot |= p->flags;
1998 /* and since the content will be modified, we must invalidate
1999 the corresponding translated code. */
2000 tb_invalidate_phys_page(addr, pc, puc, true);
2001 #ifdef DEBUG_TB_CHECK
2002 tb_invalidate_check(addr);
2003 #endif
2005 mprotect((void *)g2h(host_start), qemu_host_page_size,
2006 prot & PAGE_BITS);
2008 mmap_unlock();
2009 return 1;
2011 mmap_unlock();
2012 return 0;
2014 #endif /* CONFIG_USER_ONLY */