target-alpha: make cpu-qom.h not target specific
[qemu/ar7.git] / translate-all.c
blobb54f47253122fa64fed822f0393cb0051e211c11
1 /*
2 * Host code generation
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/mman.h>
23 #endif
24 #include "qemu/osdep.h"
27 #include "qemu-common.h"
28 #define NO_CPU_IO_DEFS
29 #include "cpu.h"
30 #include "trace.h"
31 #include "disas/disas.h"
32 #include "tcg.h"
33 #if defined(CONFIG_USER_ONLY)
34 #include "qemu.h"
35 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
36 #include <sys/param.h>
37 #if __FreeBSD_version >= 700104
38 #define HAVE_KINFO_GETVMMAP
39 #define sigqueue sigqueue_freebsd /* avoid redefinition */
40 #include <sys/proc.h>
41 #include <machine/profile.h>
42 #define _KERNEL
43 #include <sys/user.h>
44 #undef _KERNEL
45 #undef sigqueue
46 #include <libutil.h>
47 #endif
48 #endif
49 #else
50 #include "exec/address-spaces.h"
51 #endif
53 #include "exec/cputlb.h"
54 #include "exec/tb-hash.h"
55 #include "translate-all.h"
56 #include "qemu/bitmap.h"
57 #include "qemu/timer.h"
58 #include "exec/log.h"
60 //#define DEBUG_TB_INVALIDATE
61 //#define DEBUG_FLUSH
62 /* make various TB consistency checks */
63 //#define DEBUG_TB_CHECK
65 #if !defined(CONFIG_USER_ONLY)
66 /* TB consistency checks only implemented for usermode emulation. */
67 #undef DEBUG_TB_CHECK
68 #endif
70 #define SMC_BITMAP_USE_THRESHOLD 10
72 typedef struct PageDesc {
73 /* list of TBs intersecting this ram page */
74 TranslationBlock *first_tb;
75 #ifdef CONFIG_SOFTMMU
76 /* in order to optimize self modifying code, we count the number
77 of lookups we do to a given page to use a bitmap */
78 unsigned int code_write_count;
79 unsigned long *code_bitmap;
80 #else
81 unsigned long flags;
82 #endif
83 } PageDesc;
85 /* In system mode we want L1_MAP to be based on ram offsets,
86 while in user mode we want it to be based on virtual addresses. */
87 #if !defined(CONFIG_USER_ONLY)
88 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
89 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
90 #else
91 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
92 #endif
93 #else
94 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
95 #endif
97 /* Size of the L2 (and L3, etc) page tables. */
98 #define V_L2_BITS 10
99 #define V_L2_SIZE (1 << V_L2_BITS)
101 /* The bits remaining after N lower levels of page tables. */
102 #define V_L1_BITS_REM \
103 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
105 #if V_L1_BITS_REM < 4
106 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
107 #else
108 #define V_L1_BITS V_L1_BITS_REM
109 #endif
111 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
113 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
115 uintptr_t qemu_host_page_size;
116 intptr_t qemu_host_page_mask;
118 /* The bottom level has pointers to PageDesc */
119 static void *l1_map[V_L1_SIZE];
121 /* code generation context */
122 TCGContext tcg_ctx;
124 /* translation block context */
125 #ifdef CONFIG_USER_ONLY
126 __thread int have_tb_lock;
127 #endif
129 void tb_lock(void)
131 #ifdef CONFIG_USER_ONLY
132 assert(!have_tb_lock);
133 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
134 have_tb_lock++;
135 #endif
138 void tb_unlock(void)
140 #ifdef CONFIG_USER_ONLY
141 assert(have_tb_lock);
142 have_tb_lock--;
143 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
144 #endif
147 void tb_lock_reset(void)
149 #ifdef CONFIG_USER_ONLY
150 if (have_tb_lock) {
151 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
152 have_tb_lock = 0;
154 #endif
157 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
159 void cpu_gen_init(void)
161 tcg_context_init(&tcg_ctx);
164 /* Encode VAL as a signed leb128 sequence at P.
165 Return P incremented past the encoded value. */
166 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
168 int more, byte;
170 do {
171 byte = val & 0x7f;
172 val >>= 7;
173 more = !((val == 0 && (byte & 0x40) == 0)
174 || (val == -1 && (byte & 0x40) != 0));
175 if (more) {
176 byte |= 0x80;
178 *p++ = byte;
179 } while (more);
181 return p;
184 /* Decode a signed leb128 sequence at *PP; increment *PP past the
185 decoded value. Return the decoded value. */
186 static target_long decode_sleb128(uint8_t **pp)
188 uint8_t *p = *pp;
189 target_long val = 0;
190 int byte, shift = 0;
192 do {
193 byte = *p++;
194 val |= (target_ulong)(byte & 0x7f) << shift;
195 shift += 7;
196 } while (byte & 0x80);
197 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
198 val |= -(target_ulong)1 << shift;
201 *pp = p;
202 return val;
205 /* Encode the data collected about the instructions while compiling TB.
206 Place the data at BLOCK, and return the number of bytes consumed.
208 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
209 which come from the target's insn_start data, followed by a uintptr_t
210 which comes from the host pc of the end of the code implementing the insn.
212 Each line of the table is encoded as sleb128 deltas from the previous
213 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
214 That is, the first column is seeded with the guest pc, the last column
215 with the host pc, and the middle columns with zeros. */
217 static int encode_search(TranslationBlock *tb, uint8_t *block)
219 uint8_t *highwater = tcg_ctx.code_gen_highwater;
220 uint8_t *p = block;
221 int i, j, n;
223 tb->tc_search = block;
225 for (i = 0, n = tb->icount; i < n; ++i) {
226 target_ulong prev;
228 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
229 if (i == 0) {
230 prev = (j == 0 ? tb->pc : 0);
231 } else {
232 prev = tcg_ctx.gen_insn_data[i - 1][j];
234 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
236 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
237 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
239 /* Test for (pending) buffer overflow. The assumption is that any
240 one row beginning below the high water mark cannot overrun
241 the buffer completely. Thus we can test for overflow after
242 encoding a row without having to check during encoding. */
243 if (unlikely(p > highwater)) {
244 return -1;
248 return p - block;
251 /* The cpu state corresponding to 'searched_pc' is restored. */
252 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
253 uintptr_t searched_pc)
255 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
256 uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
257 CPUArchState *env = cpu->env_ptr;
258 uint8_t *p = tb->tc_search;
259 int i, j, num_insns = tb->icount;
260 #ifdef CONFIG_PROFILER
261 int64_t ti = profile_getclock();
262 #endif
264 if (searched_pc < host_pc) {
265 return -1;
268 /* Reconstruct the stored insn data while looking for the point at
269 which the end of the insn exceeds the searched_pc. */
270 for (i = 0; i < num_insns; ++i) {
271 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
272 data[j] += decode_sleb128(&p);
274 host_pc += decode_sleb128(&p);
275 if (host_pc > searched_pc) {
276 goto found;
279 return -1;
281 found:
282 if (tb->cflags & CF_USE_ICOUNT) {
283 assert(use_icount);
284 /* Reset the cycle counter to the start of the block. */
285 cpu->icount_decr.u16.low += num_insns;
286 /* Clear the IO flag. */
287 cpu->can_do_io = 0;
289 cpu->icount_decr.u16.low -= i;
290 restore_state_to_opc(env, tb, data);
292 #ifdef CONFIG_PROFILER
293 tcg_ctx.restore_time += profile_getclock() - ti;
294 tcg_ctx.restore_count++;
295 #endif
296 return 0;
299 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
301 TranslationBlock *tb;
303 tb = tb_find_pc(retaddr);
304 if (tb) {
305 cpu_restore_state_from_tb(cpu, tb, retaddr);
306 if (tb->cflags & CF_NOCACHE) {
307 /* one-shot translation, invalidate it immediately */
308 tb_phys_invalidate(tb, -1);
309 tb_free(tb);
311 return true;
313 return false;
316 void page_size_init(void)
318 /* NOTE: we can always suppose that qemu_host_page_size >=
319 TARGET_PAGE_SIZE */
320 qemu_real_host_page_size = getpagesize();
321 qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
322 if (qemu_host_page_size == 0) {
323 qemu_host_page_size = qemu_real_host_page_size;
325 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
326 qemu_host_page_size = TARGET_PAGE_SIZE;
328 qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
331 static void page_init(void)
333 page_size_init();
334 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
336 #ifdef HAVE_KINFO_GETVMMAP
337 struct kinfo_vmentry *freep;
338 int i, cnt;
340 freep = kinfo_getvmmap(getpid(), &cnt);
341 if (freep) {
342 mmap_lock();
343 for (i = 0; i < cnt; i++) {
344 unsigned long startaddr, endaddr;
346 startaddr = freep[i].kve_start;
347 endaddr = freep[i].kve_end;
348 if (h2g_valid(startaddr)) {
349 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
351 if (h2g_valid(endaddr)) {
352 endaddr = h2g(endaddr);
353 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
354 } else {
355 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
356 endaddr = ~0ul;
357 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
358 #endif
362 free(freep);
363 mmap_unlock();
365 #else
366 FILE *f;
368 last_brk = (unsigned long)sbrk(0);
370 f = fopen("/compat/linux/proc/self/maps", "r");
371 if (f) {
372 mmap_lock();
374 do {
375 unsigned long startaddr, endaddr;
376 int n;
378 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
380 if (n == 2 && h2g_valid(startaddr)) {
381 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
383 if (h2g_valid(endaddr)) {
384 endaddr = h2g(endaddr);
385 } else {
386 endaddr = ~0ul;
388 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
390 } while (!feof(f));
392 fclose(f);
393 mmap_unlock();
395 #endif
397 #endif
400 /* If alloc=1:
401 * Called with mmap_lock held for user-mode emulation.
403 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
405 PageDesc *pd;
406 void **lp;
407 int i;
409 /* Level 1. Always allocated. */
410 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
412 /* Level 2..N-1. */
413 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
414 void **p = atomic_rcu_read(lp);
416 if (p == NULL) {
417 if (!alloc) {
418 return NULL;
420 p = g_new0(void *, V_L2_SIZE);
421 atomic_rcu_set(lp, p);
424 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
427 pd = atomic_rcu_read(lp);
428 if (pd == NULL) {
429 if (!alloc) {
430 return NULL;
432 pd = g_new0(PageDesc, V_L2_SIZE);
433 atomic_rcu_set(lp, pd);
436 return pd + (index & (V_L2_SIZE - 1));
439 static inline PageDesc *page_find(tb_page_addr_t index)
441 return page_find_alloc(index, 0);
444 #if defined(CONFIG_USER_ONLY)
445 /* Currently it is not recommended to allocate big chunks of data in
446 user mode. It will change when a dedicated libc will be used. */
447 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
448 region in which the guest needs to run. Revisit this. */
449 #define USE_STATIC_CODE_GEN_BUFFER
450 #endif
452 /* Minimum size of the code gen buffer. This number is randomly chosen,
453 but not so small that we can't have a fair number of TB's live. */
454 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
456 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
457 indicated, this is constrained by the range of direct branches on the
458 host cpu, as used by the TCG implementation of goto_tb. */
459 #if defined(__x86_64__)
460 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
461 #elif defined(__sparc__)
462 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
463 #elif defined(__powerpc64__)
464 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
465 #elif defined(__powerpc__)
466 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
467 #elif defined(__aarch64__)
468 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
469 #elif defined(__arm__)
470 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
471 #elif defined(__s390x__)
472 /* We have a +- 4GB range on the branches; leave some slop. */
473 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
474 #elif defined(__mips__)
475 /* We have a 256MB branch region, but leave room to make sure the
476 main executable is also within that region. */
477 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
478 #else
479 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
480 #endif
482 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
484 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
485 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
486 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
488 static inline size_t size_code_gen_buffer(size_t tb_size)
490 /* Size the buffer. */
491 if (tb_size == 0) {
492 #ifdef USE_STATIC_CODE_GEN_BUFFER
493 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
494 #else
495 /* ??? Needs adjustments. */
496 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
497 static buffer, we could size this on RESERVED_VA, on the text
498 segment size of the executable, or continue to use the default. */
499 tb_size = (unsigned long)(ram_size / 4);
500 #endif
502 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
503 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
505 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
506 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
508 return tb_size;
511 #ifdef __mips__
512 /* In order to use J and JAL within the code_gen_buffer, we require
513 that the buffer not cross a 256MB boundary. */
514 static inline bool cross_256mb(void *addr, size_t size)
516 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
519 /* We weren't able to allocate a buffer without crossing that boundary,
520 so make do with the larger portion of the buffer that doesn't cross.
521 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
522 static inline void *split_cross_256mb(void *buf1, size_t size1)
524 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
525 size_t size2 = buf1 + size1 - buf2;
527 size1 = buf2 - buf1;
528 if (size1 < size2) {
529 size1 = size2;
530 buf1 = buf2;
533 tcg_ctx.code_gen_buffer_size = size1;
534 return buf1;
536 #endif
538 #ifdef USE_STATIC_CODE_GEN_BUFFER
539 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
540 __attribute__((aligned(CODE_GEN_ALIGN)));
542 # ifdef _WIN32
543 static inline void do_protect(void *addr, long size, int prot)
545 DWORD old_protect;
546 VirtualProtect(addr, size, prot, &old_protect);
549 static inline void map_exec(void *addr, long size)
551 do_protect(addr, size, PAGE_EXECUTE_READWRITE);
554 static inline void map_none(void *addr, long size)
556 do_protect(addr, size, PAGE_NOACCESS);
558 # else
559 static inline void do_protect(void *addr, long size, int prot)
561 uintptr_t start, end;
563 start = (uintptr_t)addr;
564 start &= qemu_real_host_page_mask;
566 end = (uintptr_t)addr + size;
567 end = ROUND_UP(end, qemu_real_host_page_size);
569 mprotect((void *)start, end - start, prot);
572 static inline void map_exec(void *addr, long size)
574 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
577 static inline void map_none(void *addr, long size)
579 do_protect(addr, size, PROT_NONE);
581 # endif /* WIN32 */
583 static inline void *alloc_code_gen_buffer(void)
585 void *buf = static_code_gen_buffer;
586 size_t full_size, size;
588 /* The size of the buffer, rounded down to end on a page boundary. */
589 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
590 & qemu_real_host_page_mask) - (uintptr_t)buf;
592 /* Reserve a guard page. */
593 size = full_size - qemu_real_host_page_size;
595 /* Honor a command-line option limiting the size of the buffer. */
596 if (size > tcg_ctx.code_gen_buffer_size) {
597 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
598 & qemu_real_host_page_mask) - (uintptr_t)buf;
600 tcg_ctx.code_gen_buffer_size = size;
602 #ifdef __mips__
603 if (cross_256mb(buf, size)) {
604 buf = split_cross_256mb(buf, size);
605 size = tcg_ctx.code_gen_buffer_size;
607 #endif
609 map_exec(buf, size);
610 map_none(buf + size, qemu_real_host_page_size);
611 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
613 return buf;
615 #elif defined(_WIN32)
616 static inline void *alloc_code_gen_buffer(void)
618 size_t size = tcg_ctx.code_gen_buffer_size;
619 void *buf1, *buf2;
621 /* Perform the allocation in two steps, so that the guard page
622 is reserved but uncommitted. */
623 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
624 MEM_RESERVE, PAGE_NOACCESS);
625 if (buf1 != NULL) {
626 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
627 assert(buf1 == buf2);
630 return buf1;
632 #else
633 static inline void *alloc_code_gen_buffer(void)
635 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
636 uintptr_t start = 0;
637 size_t size = tcg_ctx.code_gen_buffer_size;
638 void *buf;
640 /* Constrain the position of the buffer based on the host cpu.
641 Note that these addresses are chosen in concert with the
642 addresses assigned in the relevant linker script file. */
643 # if defined(__PIE__) || defined(__PIC__)
644 /* Don't bother setting a preferred location if we're building
645 a position-independent executable. We're more likely to get
646 an address near the main executable if we let the kernel
647 choose the address. */
648 # elif defined(__x86_64__) && defined(MAP_32BIT)
649 /* Force the memory down into low memory with the executable.
650 Leave the choice of exact location with the kernel. */
651 flags |= MAP_32BIT;
652 /* Cannot expect to map more than 800MB in low memory. */
653 if (size > 800u * 1024 * 1024) {
654 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
656 # elif defined(__sparc__)
657 start = 0x40000000ul;
658 # elif defined(__s390x__)
659 start = 0x90000000ul;
660 # elif defined(__mips__)
661 # if _MIPS_SIM == _ABI64
662 start = 0x128000000ul;
663 # else
664 start = 0x08000000ul;
665 # endif
666 # endif
668 buf = mmap((void *)start, size + qemu_real_host_page_size,
669 PROT_NONE, flags, -1, 0);
670 if (buf == MAP_FAILED) {
671 return NULL;
674 #ifdef __mips__
675 if (cross_256mb(buf, size)) {
676 /* Try again, with the original still mapped, to avoid re-acquiring
677 that 256mb crossing. This time don't specify an address. */
678 size_t size2;
679 void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
680 PROT_NONE, flags, -1, 0);
681 switch (buf2 != MAP_FAILED) {
682 case 1:
683 if (!cross_256mb(buf2, size)) {
684 /* Success! Use the new buffer. */
685 munmap(buf, size + qemu_real_host_page_size);
686 break;
688 /* Failure. Work with what we had. */
689 munmap(buf2, size + qemu_real_host_page_size);
690 /* fallthru */
691 default:
692 /* Split the original buffer. Free the smaller half. */
693 buf2 = split_cross_256mb(buf, size);
694 size2 = tcg_ctx.code_gen_buffer_size;
695 if (buf == buf2) {
696 munmap(buf + size2 + qemu_real_host_page_size, size - size2);
697 } else {
698 munmap(buf, size - size2);
700 size = size2;
701 break;
703 buf = buf2;
705 #endif
707 /* Make the final buffer accessible. The guard page at the end
708 will remain inaccessible with PROT_NONE. */
709 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
711 /* Request large pages for the buffer. */
712 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
714 return buf;
716 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
718 static inline void code_gen_alloc(size_t tb_size)
720 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
721 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
722 if (tcg_ctx.code_gen_buffer == NULL) {
723 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
724 exit(1);
727 /* Estimate a good size for the number of TBs we can support. We
728 still haven't deducted the prologue from the buffer size here,
729 but that's minimal and won't affect the estimate much. */
730 tcg_ctx.code_gen_max_blocks
731 = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
732 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
734 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
737 /* Must be called before using the QEMU cpus. 'tb_size' is the size
738 (in bytes) allocated to the translation buffer. Zero means default
739 size. */
740 void tcg_exec_init(unsigned long tb_size)
742 cpu_gen_init();
743 page_init();
744 code_gen_alloc(tb_size);
745 #if defined(CONFIG_SOFTMMU)
746 /* There's no guest base to take into account, so go ahead and
747 initialize the prologue now. */
748 tcg_prologue_init(&tcg_ctx);
749 #endif
752 bool tcg_enabled(void)
754 return tcg_ctx.code_gen_buffer != NULL;
757 /* Allocate a new translation block. Flush the translation buffer if
758 too many translation blocks or too much generated code. */
759 static TranslationBlock *tb_alloc(target_ulong pc)
761 TranslationBlock *tb;
763 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
764 return NULL;
766 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
767 tb->pc = pc;
768 tb->cflags = 0;
769 return tb;
772 void tb_free(TranslationBlock *tb)
774 /* In practice this is mostly used for single use temporary TB
775 Ignore the hard cases and just back up if this TB happens to
776 be the last one generated. */
777 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
778 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
779 tcg_ctx.code_gen_ptr = tb->tc_ptr;
780 tcg_ctx.tb_ctx.nb_tbs--;
784 static inline void invalidate_page_bitmap(PageDesc *p)
786 #ifdef CONFIG_SOFTMMU
787 g_free(p->code_bitmap);
788 p->code_bitmap = NULL;
789 p->code_write_count = 0;
790 #endif
793 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
794 static void page_flush_tb_1(int level, void **lp)
796 int i;
798 if (*lp == NULL) {
799 return;
801 if (level == 0) {
802 PageDesc *pd = *lp;
804 for (i = 0; i < V_L2_SIZE; ++i) {
805 pd[i].first_tb = NULL;
806 invalidate_page_bitmap(pd + i);
808 } else {
809 void **pp = *lp;
811 for (i = 0; i < V_L2_SIZE; ++i) {
812 page_flush_tb_1(level - 1, pp + i);
817 static void page_flush_tb(void)
819 int i;
821 for (i = 0; i < V_L1_SIZE; i++) {
822 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
826 /* flush all the translation blocks */
827 /* XXX: tb_flush is currently not thread safe */
828 void tb_flush(CPUState *cpu)
830 #if defined(DEBUG_FLUSH)
831 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
832 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
833 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
834 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
835 tcg_ctx.tb_ctx.nb_tbs : 0);
836 #endif
837 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
838 > tcg_ctx.code_gen_buffer_size) {
839 cpu_abort(cpu, "Internal error: code buffer overflow\n");
841 tcg_ctx.tb_ctx.nb_tbs = 0;
843 CPU_FOREACH(cpu) {
844 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
845 cpu->tb_flushed = true;
848 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
849 page_flush_tb();
851 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
852 /* XXX: flush processor icache at this point if cache flush is
853 expensive */
854 tcg_ctx.tb_ctx.tb_flush_count++;
857 #ifdef DEBUG_TB_CHECK
859 static void tb_invalidate_check(target_ulong address)
861 TranslationBlock *tb;
862 int i;
864 address &= TARGET_PAGE_MASK;
865 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
866 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
867 tb = tb->phys_hash_next) {
868 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
869 address >= tb->pc + tb->size)) {
870 printf("ERROR invalidate: address=" TARGET_FMT_lx
871 " PC=%08lx size=%04x\n",
872 address, (long)tb->pc, tb->size);
878 /* verify that all the pages have correct rights for code */
879 static void tb_page_check(void)
881 TranslationBlock *tb;
882 int i, flags1, flags2;
884 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
885 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
886 tb = tb->phys_hash_next) {
887 flags1 = page_get_flags(tb->pc);
888 flags2 = page_get_flags(tb->pc + tb->size - 1);
889 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
890 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
891 (long)tb->pc, tb->size, flags1, flags2);
897 #endif
899 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
901 TranslationBlock *tb1;
903 for (;;) {
904 tb1 = *ptb;
905 if (tb1 == tb) {
906 *ptb = tb1->phys_hash_next;
907 break;
909 ptb = &tb1->phys_hash_next;
913 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
915 TranslationBlock *tb1;
916 unsigned int n1;
918 for (;;) {
919 tb1 = *ptb;
920 n1 = (uintptr_t)tb1 & 3;
921 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
922 if (tb1 == tb) {
923 *ptb = tb1->page_next[n1];
924 break;
926 ptb = &tb1->page_next[n1];
930 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
931 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
933 TranslationBlock *tb1;
934 uintptr_t *ptb, ntb;
935 unsigned int n1;
937 ptb = &tb->jmp_list_next[n];
938 if (*ptb) {
939 /* find tb(n) in circular list */
940 for (;;) {
941 ntb = *ptb;
942 n1 = ntb & 3;
943 tb1 = (TranslationBlock *)(ntb & ~3);
944 if (n1 == n && tb1 == tb) {
945 break;
947 if (n1 == 2) {
948 ptb = &tb1->jmp_list_first;
949 } else {
950 ptb = &tb1->jmp_list_next[n1];
953 /* now we can suppress tb(n) from the list */
954 *ptb = tb->jmp_list_next[n];
956 tb->jmp_list_next[n] = (uintptr_t)NULL;
960 /* reset the jump entry 'n' of a TB so that it is not chained to
961 another TB */
962 static inline void tb_reset_jump(TranslationBlock *tb, int n)
964 uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
965 tb_set_jmp_target(tb, n, addr);
968 /* remove any jumps to the TB */
969 static inline void tb_jmp_unlink(TranslationBlock *tb)
971 TranslationBlock *tb1;
972 uintptr_t *ptb, ntb;
973 unsigned int n1;
975 ptb = &tb->jmp_list_first;
976 for (;;) {
977 ntb = *ptb;
978 n1 = ntb & 3;
979 tb1 = (TranslationBlock *)(ntb & ~3);
980 if (n1 == 2) {
981 break;
983 tb_reset_jump(tb1, n1);
984 *ptb = tb1->jmp_list_next[n1];
985 tb1->jmp_list_next[n1] = (uintptr_t)NULL;
989 /* invalidate one TB */
990 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
992 CPUState *cpu;
993 PageDesc *p;
994 unsigned int h;
995 tb_page_addr_t phys_pc;
997 /* remove the TB from the hash list */
998 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
999 h = tb_phys_hash_func(phys_pc);
1000 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
1002 /* remove the TB from the page list */
1003 if (tb->page_addr[0] != page_addr) {
1004 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1005 tb_page_remove(&p->first_tb, tb);
1006 invalidate_page_bitmap(p);
1008 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
1009 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1010 tb_page_remove(&p->first_tb, tb);
1011 invalidate_page_bitmap(p);
1014 /* remove the TB from the hash list */
1015 h = tb_jmp_cache_hash_func(tb->pc);
1016 CPU_FOREACH(cpu) {
1017 if (cpu->tb_jmp_cache[h] == tb) {
1018 cpu->tb_jmp_cache[h] = NULL;
1022 /* suppress this TB from the two jump lists */
1023 tb_remove_from_jmp_list(tb, 0);
1024 tb_remove_from_jmp_list(tb, 1);
1026 /* suppress any remaining jumps to this TB */
1027 tb_jmp_unlink(tb);
1029 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
1032 #ifdef CONFIG_SOFTMMU
1033 static void build_page_bitmap(PageDesc *p)
1035 int n, tb_start, tb_end;
1036 TranslationBlock *tb;
1038 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1040 tb = p->first_tb;
1041 while (tb != NULL) {
1042 n = (uintptr_t)tb & 3;
1043 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1044 /* NOTE: this is subtle as a TB may span two physical pages */
1045 if (n == 0) {
1046 /* NOTE: tb_end may be after the end of the page, but
1047 it is not a problem */
1048 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1049 tb_end = tb_start + tb->size;
1050 if (tb_end > TARGET_PAGE_SIZE) {
1051 tb_end = TARGET_PAGE_SIZE;
1053 } else {
1054 tb_start = 0;
1055 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1057 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1058 tb = tb->page_next[n];
1061 #endif
1063 /* add the tb in the target page and protect it if necessary
1065 * Called with mmap_lock held for user-mode emulation.
1067 static inline void tb_alloc_page(TranslationBlock *tb,
1068 unsigned int n, tb_page_addr_t page_addr)
1070 PageDesc *p;
1071 #ifndef CONFIG_USER_ONLY
1072 bool page_already_protected;
1073 #endif
1075 tb->page_addr[n] = page_addr;
1076 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1077 tb->page_next[n] = p->first_tb;
1078 #ifndef CONFIG_USER_ONLY
1079 page_already_protected = p->first_tb != NULL;
1080 #endif
1081 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1082 invalidate_page_bitmap(p);
1084 #if defined(CONFIG_USER_ONLY)
1085 if (p->flags & PAGE_WRITE) {
1086 target_ulong addr;
1087 PageDesc *p2;
1088 int prot;
1090 /* force the host page as non writable (writes will have a
1091 page fault + mprotect overhead) */
1092 page_addr &= qemu_host_page_mask;
1093 prot = 0;
1094 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1095 addr += TARGET_PAGE_SIZE) {
1097 p2 = page_find(addr >> TARGET_PAGE_BITS);
1098 if (!p2) {
1099 continue;
1101 prot |= p2->flags;
1102 p2->flags &= ~PAGE_WRITE;
1104 mprotect(g2h(page_addr), qemu_host_page_size,
1105 (prot & PAGE_BITS) & ~PAGE_WRITE);
1106 #ifdef DEBUG_TB_INVALIDATE
1107 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1108 page_addr);
1109 #endif
1111 #else
1112 /* if some code is already present, then the pages are already
1113 protected. So we handle the case where only the first TB is
1114 allocated in a physical page */
1115 if (!page_already_protected) {
1116 tlb_protect_code(page_addr);
1118 #endif
1121 /* add a new TB and link it to the physical page tables. phys_page2 is
1122 * (-1) to indicate that only one page contains the TB.
1124 * Called with mmap_lock held for user-mode emulation.
1126 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1127 tb_page_addr_t phys_page2)
1129 unsigned int h;
1130 TranslationBlock **ptb;
1132 /* add in the physical hash table */
1133 h = tb_phys_hash_func(phys_pc);
1134 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1135 tb->phys_hash_next = *ptb;
1136 *ptb = tb;
1138 /* add in the page list */
1139 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1140 if (phys_page2 != -1) {
1141 tb_alloc_page(tb, 1, phys_page2);
1142 } else {
1143 tb->page_addr[1] = -1;
1146 #ifdef DEBUG_TB_CHECK
1147 tb_page_check();
1148 #endif
1151 /* Called with mmap_lock held for user mode emulation. */
1152 TranslationBlock *tb_gen_code(CPUState *cpu,
1153 target_ulong pc, target_ulong cs_base,
1154 uint32_t flags, int cflags)
1156 CPUArchState *env = cpu->env_ptr;
1157 TranslationBlock *tb;
1158 tb_page_addr_t phys_pc, phys_page2;
1159 target_ulong virt_page2;
1160 tcg_insn_unit *gen_code_buf;
1161 int gen_code_size, search_size;
1162 #ifdef CONFIG_PROFILER
1163 int64_t ti;
1164 #endif
1166 phys_pc = get_page_addr_code(env, pc);
1167 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
1168 cflags |= CF_USE_ICOUNT;
1171 tb = tb_alloc(pc);
1172 if (unlikely(!tb)) {
1173 buffer_overflow:
1174 /* flush must be done */
1175 tb_flush(cpu);
1176 /* cannot fail at this point */
1177 tb = tb_alloc(pc);
1178 assert(tb != NULL);
1181 gen_code_buf = tcg_ctx.code_gen_ptr;
1182 tb->tc_ptr = gen_code_buf;
1183 tb->cs_base = cs_base;
1184 tb->flags = flags;
1185 tb->cflags = cflags;
1187 #ifdef CONFIG_PROFILER
1188 tcg_ctx.tb_count1++; /* includes aborted translations because of
1189 exceptions */
1190 ti = profile_getclock();
1191 #endif
1193 tcg_func_start(&tcg_ctx);
1195 gen_intermediate_code(env, tb);
1197 trace_translate_block(tb, tb->pc, tb->tc_ptr);
1199 /* generate machine code */
1200 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1201 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1202 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
1203 #ifdef USE_DIRECT_JUMP
1204 tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
1205 tcg_ctx.tb_jmp_target_addr = NULL;
1206 #else
1207 tcg_ctx.tb_jmp_insn_offset = NULL;
1208 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;
1209 #endif
1211 #ifdef CONFIG_PROFILER
1212 tcg_ctx.tb_count++;
1213 tcg_ctx.interm_time += profile_getclock() - ti;
1214 tcg_ctx.code_time -= profile_getclock();
1215 #endif
1217 /* ??? Overflow could be handled better here. In particular, we
1218 don't need to re-do gen_intermediate_code, nor should we re-do
1219 the tcg optimization currently hidden inside tcg_gen_code. All
1220 that should be required is to flush the TBs, allocate a new TB,
1221 re-initialize it per above, and re-do the actual code generation. */
1222 gen_code_size = tcg_gen_code(&tcg_ctx, tb);
1223 if (unlikely(gen_code_size < 0)) {
1224 goto buffer_overflow;
1226 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1227 if (unlikely(search_size < 0)) {
1228 goto buffer_overflow;
1231 #ifdef CONFIG_PROFILER
1232 tcg_ctx.code_time += profile_getclock();
1233 tcg_ctx.code_in_len += tb->size;
1234 tcg_ctx.code_out_len += gen_code_size;
1235 tcg_ctx.search_out_len += search_size;
1236 #endif
1238 #ifdef DEBUG_DISAS
1239 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1240 qemu_log_in_addr_range(tb->pc)) {
1241 qemu_log("OUT: [size=%d]\n", gen_code_size);
1242 log_disas(tb->tc_ptr, gen_code_size);
1243 qemu_log("\n");
1244 qemu_log_flush();
1246 #endif
1248 tcg_ctx.code_gen_ptr = (void *)
1249 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1250 CODE_GEN_ALIGN);
1252 /* init jump list */
1253 assert(((uintptr_t)tb & 3) == 0);
1254 tb->jmp_list_first = (uintptr_t)tb | 2;
1255 tb->jmp_list_next[0] = (uintptr_t)NULL;
1256 tb->jmp_list_next[1] = (uintptr_t)NULL;
1258 /* init original jump addresses wich has been set during tcg_gen_code() */
1259 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1260 tb_reset_jump(tb, 0);
1262 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1263 tb_reset_jump(tb, 1);
1266 /* check next page if needed */
1267 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1268 phys_page2 = -1;
1269 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1270 phys_page2 = get_page_addr_code(env, virt_page2);
1272 /* As long as consistency of the TB stuff is provided by tb_lock in user
1273 * mode and is implicit in single-threaded softmmu emulation, no explicit
1274 * memory barrier is required before tb_link_page() makes the TB visible
1275 * through the physical hash table and physical page list.
1277 tb_link_page(tb, phys_pc, phys_page2);
1278 return tb;
1282 * Invalidate all TBs which intersect with the target physical address range
1283 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1284 * 'is_cpu_write_access' should be true if called from a real cpu write
1285 * access: the virtual CPU will exit the current TB if code is modified inside
1286 * this TB.
1288 * Called with mmap_lock held for user-mode emulation
1290 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1292 while (start < end) {
1293 tb_invalidate_phys_page_range(start, end, 0);
1294 start &= TARGET_PAGE_MASK;
1295 start += TARGET_PAGE_SIZE;
1300 * Invalidate all TBs which intersect with the target physical address range
1301 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1302 * 'is_cpu_write_access' should be true if called from a real cpu write
1303 * access: the virtual CPU will exit the current TB if code is modified inside
1304 * this TB.
1306 * Called with mmap_lock held for user-mode emulation
1308 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1309 int is_cpu_write_access)
1311 TranslationBlock *tb, *tb_next;
1312 #if defined(TARGET_HAS_PRECISE_SMC)
1313 CPUState *cpu = current_cpu;
1314 CPUArchState *env = NULL;
1315 #endif
1316 tb_page_addr_t tb_start, tb_end;
1317 PageDesc *p;
1318 int n;
1319 #ifdef TARGET_HAS_PRECISE_SMC
1320 int current_tb_not_found = is_cpu_write_access;
1321 TranslationBlock *current_tb = NULL;
1322 int current_tb_modified = 0;
1323 target_ulong current_pc = 0;
1324 target_ulong current_cs_base = 0;
1325 uint32_t current_flags = 0;
1326 #endif /* TARGET_HAS_PRECISE_SMC */
1328 p = page_find(start >> TARGET_PAGE_BITS);
1329 if (!p) {
1330 return;
1332 #if defined(TARGET_HAS_PRECISE_SMC)
1333 if (cpu != NULL) {
1334 env = cpu->env_ptr;
1336 #endif
1338 /* we remove all the TBs in the range [start, end[ */
1339 /* XXX: see if in some cases it could be faster to invalidate all
1340 the code */
1341 tb = p->first_tb;
1342 while (tb != NULL) {
1343 n = (uintptr_t)tb & 3;
1344 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1345 tb_next = tb->page_next[n];
1346 /* NOTE: this is subtle as a TB may span two physical pages */
1347 if (n == 0) {
1348 /* NOTE: tb_end may be after the end of the page, but
1349 it is not a problem */
1350 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1351 tb_end = tb_start + tb->size;
1352 } else {
1353 tb_start = tb->page_addr[1];
1354 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1356 if (!(tb_end <= start || tb_start >= end)) {
1357 #ifdef TARGET_HAS_PRECISE_SMC
1358 if (current_tb_not_found) {
1359 current_tb_not_found = 0;
1360 current_tb = NULL;
1361 if (cpu->mem_io_pc) {
1362 /* now we have a real cpu fault */
1363 current_tb = tb_find_pc(cpu->mem_io_pc);
1366 if (current_tb == tb &&
1367 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1368 /* If we are modifying the current TB, we must stop
1369 its execution. We could be more precise by checking
1370 that the modification is after the current PC, but it
1371 would require a specialized function to partially
1372 restore the CPU state */
1374 current_tb_modified = 1;
1375 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1376 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1377 &current_flags);
1379 #endif /* TARGET_HAS_PRECISE_SMC */
1380 tb_phys_invalidate(tb, -1);
1382 tb = tb_next;
1384 #if !defined(CONFIG_USER_ONLY)
1385 /* if no code remaining, no need to continue to use slow writes */
1386 if (!p->first_tb) {
1387 invalidate_page_bitmap(p);
1388 tlb_unprotect_code(start);
1390 #endif
1391 #ifdef TARGET_HAS_PRECISE_SMC
1392 if (current_tb_modified) {
1393 /* we generate a block containing just the instruction
1394 modifying the memory. It will ensure that it cannot modify
1395 itself */
1396 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1397 cpu_resume_from_signal(cpu, NULL);
1399 #endif
1402 #ifdef CONFIG_SOFTMMU
1403 /* len must be <= 8 and start must be a multiple of len */
1404 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1406 PageDesc *p;
1408 #if 0
1409 if (1) {
1410 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1411 cpu_single_env->mem_io_vaddr, len,
1412 cpu_single_env->eip,
1413 cpu_single_env->eip +
1414 (intptr_t)cpu_single_env->segs[R_CS].base);
1416 #endif
1417 p = page_find(start >> TARGET_PAGE_BITS);
1418 if (!p) {
1419 return;
1421 if (!p->code_bitmap &&
1422 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1423 /* build code bitmap */
1424 build_page_bitmap(p);
1426 if (p->code_bitmap) {
1427 unsigned int nr;
1428 unsigned long b;
1430 nr = start & ~TARGET_PAGE_MASK;
1431 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1432 if (b & ((1 << len) - 1)) {
1433 goto do_invalidate;
1435 } else {
1436 do_invalidate:
1437 tb_invalidate_phys_page_range(start, start + len, 1);
1440 #else
1441 /* Called with mmap_lock held. */
1442 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1443 uintptr_t pc, void *puc,
1444 bool locked)
1446 TranslationBlock *tb;
1447 PageDesc *p;
1448 int n;
1449 #ifdef TARGET_HAS_PRECISE_SMC
1450 TranslationBlock *current_tb = NULL;
1451 CPUState *cpu = current_cpu;
1452 CPUArchState *env = NULL;
1453 int current_tb_modified = 0;
1454 target_ulong current_pc = 0;
1455 target_ulong current_cs_base = 0;
1456 uint32_t current_flags = 0;
1457 #endif
1459 addr &= TARGET_PAGE_MASK;
1460 p = page_find(addr >> TARGET_PAGE_BITS);
1461 if (!p) {
1462 return;
1464 tb = p->first_tb;
1465 #ifdef TARGET_HAS_PRECISE_SMC
1466 if (tb && pc != 0) {
1467 current_tb = tb_find_pc(pc);
1469 if (cpu != NULL) {
1470 env = cpu->env_ptr;
1472 #endif
1473 while (tb != NULL) {
1474 n = (uintptr_t)tb & 3;
1475 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1476 #ifdef TARGET_HAS_PRECISE_SMC
1477 if (current_tb == tb &&
1478 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1479 /* If we are modifying the current TB, we must stop
1480 its execution. We could be more precise by checking
1481 that the modification is after the current PC, but it
1482 would require a specialized function to partially
1483 restore the CPU state */
1485 current_tb_modified = 1;
1486 cpu_restore_state_from_tb(cpu, current_tb, pc);
1487 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1488 &current_flags);
1490 #endif /* TARGET_HAS_PRECISE_SMC */
1491 tb_phys_invalidate(tb, addr);
1492 tb = tb->page_next[n];
1494 p->first_tb = NULL;
1495 #ifdef TARGET_HAS_PRECISE_SMC
1496 if (current_tb_modified) {
1497 /* we generate a block containing just the instruction
1498 modifying the memory. It will ensure that it cannot modify
1499 itself */
1500 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1501 if (locked) {
1502 mmap_unlock();
1504 cpu_resume_from_signal(cpu, puc);
1506 #endif
1508 #endif
1510 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1511 tb[1].tc_ptr. Return NULL if not found */
1512 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1514 int m_min, m_max, m;
1515 uintptr_t v;
1516 TranslationBlock *tb;
1518 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1519 return NULL;
1521 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1522 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1523 return NULL;
1525 /* binary search (cf Knuth) */
1526 m_min = 0;
1527 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1528 while (m_min <= m_max) {
1529 m = (m_min + m_max) >> 1;
1530 tb = &tcg_ctx.tb_ctx.tbs[m];
1531 v = (uintptr_t)tb->tc_ptr;
1532 if (v == tc_ptr) {
1533 return tb;
1534 } else if (tc_ptr < v) {
1535 m_max = m - 1;
1536 } else {
1537 m_min = m + 1;
1540 return &tcg_ctx.tb_ctx.tbs[m_max];
1543 #if !defined(CONFIG_USER_ONLY)
1544 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1546 ram_addr_t ram_addr;
1547 MemoryRegion *mr;
1548 hwaddr l = 1;
1550 rcu_read_lock();
1551 mr = address_space_translate(as, addr, &addr, &l, false);
1552 if (!(memory_region_is_ram(mr)
1553 || memory_region_is_romd(mr))) {
1554 rcu_read_unlock();
1555 return;
1557 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1558 + addr;
1559 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1560 rcu_read_unlock();
1562 #endif /* !defined(CONFIG_USER_ONLY) */
1564 void tb_check_watchpoint(CPUState *cpu)
1566 TranslationBlock *tb;
1568 tb = tb_find_pc(cpu->mem_io_pc);
1569 if (tb) {
1570 /* We can use retranslation to find the PC. */
1571 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1572 tb_phys_invalidate(tb, -1);
1573 } else {
1574 /* The exception probably happened in a helper. The CPU state should
1575 have been saved before calling it. Fetch the PC from there. */
1576 CPUArchState *env = cpu->env_ptr;
1577 target_ulong pc, cs_base;
1578 tb_page_addr_t addr;
1579 uint32_t flags;
1581 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1582 addr = get_page_addr_code(env, pc);
1583 tb_invalidate_phys_range(addr, addr + 1);
1587 #ifndef CONFIG_USER_ONLY
1588 /* in deterministic execution mode, instructions doing device I/Os
1589 must be at the end of the TB */
1590 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1592 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1593 CPUArchState *env = cpu->env_ptr;
1594 #endif
1595 TranslationBlock *tb;
1596 uint32_t n, cflags;
1597 target_ulong pc, cs_base;
1598 uint32_t flags;
1600 tb = tb_find_pc(retaddr);
1601 if (!tb) {
1602 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1603 (void *)retaddr);
1605 n = cpu->icount_decr.u16.low + tb->icount;
1606 cpu_restore_state_from_tb(cpu, tb, retaddr);
1607 /* Calculate how many instructions had been executed before the fault
1608 occurred. */
1609 n = n - cpu->icount_decr.u16.low;
1610 /* Generate a new TB ending on the I/O insn. */
1611 n++;
1612 /* On MIPS and SH, delay slot instructions can only be restarted if
1613 they were already the first instruction in the TB. If this is not
1614 the first instruction in a TB then re-execute the preceding
1615 branch. */
1616 #if defined(TARGET_MIPS)
1617 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1618 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1619 cpu->icount_decr.u16.low++;
1620 env->hflags &= ~MIPS_HFLAG_BMASK;
1622 #elif defined(TARGET_SH4)
1623 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1624 && n > 1) {
1625 env->pc -= 2;
1626 cpu->icount_decr.u16.low++;
1627 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1629 #endif
1630 /* This should never happen. */
1631 if (n > CF_COUNT_MASK) {
1632 cpu_abort(cpu, "TB too big during recompile");
1635 cflags = n | CF_LAST_IO;
1636 pc = tb->pc;
1637 cs_base = tb->cs_base;
1638 flags = tb->flags;
1639 tb_phys_invalidate(tb, -1);
1640 if (tb->cflags & CF_NOCACHE) {
1641 if (tb->orig_tb) {
1642 /* Invalidate original TB if this TB was generated in
1643 * cpu_exec_nocache() */
1644 tb_phys_invalidate(tb->orig_tb, -1);
1646 tb_free(tb);
1648 /* FIXME: In theory this could raise an exception. In practice
1649 we have already translated the block once so it's probably ok. */
1650 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1651 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1652 the first in the TB) then we end up generating a whole new TB and
1653 repeating the fault, which is horribly inefficient.
1654 Better would be to execute just this insn uncached, or generate a
1655 second new TB. */
1656 cpu_resume_from_signal(cpu, NULL);
1659 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1661 unsigned int i;
1663 /* Discard jump cache entries for any tb which might potentially
1664 overlap the flushed page. */
1665 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1666 memset(&cpu->tb_jmp_cache[i], 0,
1667 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1669 i = tb_jmp_cache_hash_page(addr);
1670 memset(&cpu->tb_jmp_cache[i], 0,
1671 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1674 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1676 int i, target_code_size, max_target_code_size;
1677 int direct_jmp_count, direct_jmp2_count, cross_page;
1678 TranslationBlock *tb;
1680 target_code_size = 0;
1681 max_target_code_size = 0;
1682 cross_page = 0;
1683 direct_jmp_count = 0;
1684 direct_jmp2_count = 0;
1685 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1686 tb = &tcg_ctx.tb_ctx.tbs[i];
1687 target_code_size += tb->size;
1688 if (tb->size > max_target_code_size) {
1689 max_target_code_size = tb->size;
1691 if (tb->page_addr[1] != -1) {
1692 cross_page++;
1694 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1695 direct_jmp_count++;
1696 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1697 direct_jmp2_count++;
1701 /* XXX: avoid using doubles ? */
1702 cpu_fprintf(f, "Translation buffer state:\n");
1703 cpu_fprintf(f, "gen code size %td/%zd\n",
1704 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1705 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
1706 cpu_fprintf(f, "TB count %d/%d\n",
1707 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1708 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1709 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1710 tcg_ctx.tb_ctx.nb_tbs : 0,
1711 max_target_code_size);
1712 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1713 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1714 tcg_ctx.code_gen_buffer) /
1715 tcg_ctx.tb_ctx.nb_tbs : 0,
1716 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1717 tcg_ctx.code_gen_buffer) /
1718 target_code_size : 0);
1719 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1720 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1721 tcg_ctx.tb_ctx.nb_tbs : 0);
1722 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1723 direct_jmp_count,
1724 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1725 tcg_ctx.tb_ctx.nb_tbs : 0,
1726 direct_jmp2_count,
1727 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1728 tcg_ctx.tb_ctx.nb_tbs : 0);
1729 cpu_fprintf(f, "\nStatistics:\n");
1730 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1731 cpu_fprintf(f, "TB invalidate count %d\n",
1732 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1733 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1734 tcg_dump_info(f, cpu_fprintf);
1737 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1739 tcg_dump_op_count(f, cpu_fprintf);
1742 #else /* CONFIG_USER_ONLY */
1744 void cpu_interrupt(CPUState *cpu, int mask)
1746 cpu->interrupt_request |= mask;
1747 cpu->tcg_exit_req = 1;
1751 * Walks guest process memory "regions" one by one
1752 * and calls callback function 'fn' for each region.
1754 struct walk_memory_regions_data {
1755 walk_memory_regions_fn fn;
1756 void *priv;
1757 target_ulong start;
1758 int prot;
1761 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1762 target_ulong end, int new_prot)
1764 if (data->start != -1u) {
1765 int rc = data->fn(data->priv, data->start, end, data->prot);
1766 if (rc != 0) {
1767 return rc;
1771 data->start = (new_prot ? end : -1u);
1772 data->prot = new_prot;
1774 return 0;
1777 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1778 target_ulong base, int level, void **lp)
1780 target_ulong pa;
1781 int i, rc;
1783 if (*lp == NULL) {
1784 return walk_memory_regions_end(data, base, 0);
1787 if (level == 0) {
1788 PageDesc *pd = *lp;
1790 for (i = 0; i < V_L2_SIZE; ++i) {
1791 int prot = pd[i].flags;
1793 pa = base | (i << TARGET_PAGE_BITS);
1794 if (prot != data->prot) {
1795 rc = walk_memory_regions_end(data, pa, prot);
1796 if (rc != 0) {
1797 return rc;
1801 } else {
1802 void **pp = *lp;
1804 for (i = 0; i < V_L2_SIZE; ++i) {
1805 pa = base | ((target_ulong)i <<
1806 (TARGET_PAGE_BITS + V_L2_BITS * level));
1807 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1808 if (rc != 0) {
1809 return rc;
1814 return 0;
1817 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1819 struct walk_memory_regions_data data;
1820 uintptr_t i;
1822 data.fn = fn;
1823 data.priv = priv;
1824 data.start = -1u;
1825 data.prot = 0;
1827 for (i = 0; i < V_L1_SIZE; i++) {
1828 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
1829 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1830 if (rc != 0) {
1831 return rc;
1835 return walk_memory_regions_end(&data, 0, 0);
1838 static int dump_region(void *priv, target_ulong start,
1839 target_ulong end, unsigned long prot)
1841 FILE *f = (FILE *)priv;
1843 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1844 " "TARGET_FMT_lx" %c%c%c\n",
1845 start, end, end - start,
1846 ((prot & PAGE_READ) ? 'r' : '-'),
1847 ((prot & PAGE_WRITE) ? 'w' : '-'),
1848 ((prot & PAGE_EXEC) ? 'x' : '-'));
1850 return 0;
1853 /* dump memory mappings */
1854 void page_dump(FILE *f)
1856 const int length = sizeof(target_ulong) * 2;
1857 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1858 length, "start", length, "end", length, "size", "prot");
1859 walk_memory_regions(f, dump_region);
1862 int page_get_flags(target_ulong address)
1864 PageDesc *p;
1866 p = page_find(address >> TARGET_PAGE_BITS);
1867 if (!p) {
1868 return 0;
1870 return p->flags;
1873 /* Modify the flags of a page and invalidate the code if necessary.
1874 The flag PAGE_WRITE_ORG is positioned automatically depending
1875 on PAGE_WRITE. The mmap_lock should already be held. */
1876 void page_set_flags(target_ulong start, target_ulong end, int flags)
1878 target_ulong addr, len;
1880 /* This function should never be called with addresses outside the
1881 guest address space. If this assert fires, it probably indicates
1882 a missing call to h2g_valid. */
1883 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1884 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1885 #endif
1886 assert(start < end);
1888 start = start & TARGET_PAGE_MASK;
1889 end = TARGET_PAGE_ALIGN(end);
1891 if (flags & PAGE_WRITE) {
1892 flags |= PAGE_WRITE_ORG;
1895 for (addr = start, len = end - start;
1896 len != 0;
1897 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1898 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1900 /* If the write protection bit is set, then we invalidate
1901 the code inside. */
1902 if (!(p->flags & PAGE_WRITE) &&
1903 (flags & PAGE_WRITE) &&
1904 p->first_tb) {
1905 tb_invalidate_phys_page(addr, 0, NULL, false);
1907 p->flags = flags;
1911 int page_check_range(target_ulong start, target_ulong len, int flags)
1913 PageDesc *p;
1914 target_ulong end;
1915 target_ulong addr;
1917 /* This function should never be called with addresses outside the
1918 guest address space. If this assert fires, it probably indicates
1919 a missing call to h2g_valid. */
1920 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1921 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1922 #endif
1924 if (len == 0) {
1925 return 0;
1927 if (start + len - 1 < start) {
1928 /* We've wrapped around. */
1929 return -1;
1932 /* must do before we loose bits in the next step */
1933 end = TARGET_PAGE_ALIGN(start + len);
1934 start = start & TARGET_PAGE_MASK;
1936 for (addr = start, len = end - start;
1937 len != 0;
1938 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1939 p = page_find(addr >> TARGET_PAGE_BITS);
1940 if (!p) {
1941 return -1;
1943 if (!(p->flags & PAGE_VALID)) {
1944 return -1;
1947 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1948 return -1;
1950 if (flags & PAGE_WRITE) {
1951 if (!(p->flags & PAGE_WRITE_ORG)) {
1952 return -1;
1954 /* unprotect the page if it was put read-only because it
1955 contains translated code */
1956 if (!(p->flags & PAGE_WRITE)) {
1957 if (!page_unprotect(addr, 0, NULL)) {
1958 return -1;
1963 return 0;
1966 /* called from signal handler: invalidate the code and unprotect the
1967 page. Return TRUE if the fault was successfully handled. */
1968 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1970 unsigned int prot;
1971 PageDesc *p;
1972 target_ulong host_start, host_end, addr;
1974 /* Technically this isn't safe inside a signal handler. However we
1975 know this only ever happens in a synchronous SEGV handler, so in
1976 practice it seems to be ok. */
1977 mmap_lock();
1979 p = page_find(address >> TARGET_PAGE_BITS);
1980 if (!p) {
1981 mmap_unlock();
1982 return 0;
1985 /* if the page was really writable, then we change its
1986 protection back to writable */
1987 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1988 host_start = address & qemu_host_page_mask;
1989 host_end = host_start + qemu_host_page_size;
1991 prot = 0;
1992 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1993 p = page_find(addr >> TARGET_PAGE_BITS);
1994 p->flags |= PAGE_WRITE;
1995 prot |= p->flags;
1997 /* and since the content will be modified, we must invalidate
1998 the corresponding translated code. */
1999 tb_invalidate_phys_page(addr, pc, puc, true);
2000 #ifdef DEBUG_TB_CHECK
2001 tb_invalidate_check(addr);
2002 #endif
2004 mprotect((void *)g2h(host_start), qemu_host_page_size,
2005 prot & PAGE_BITS);
2007 mmap_unlock();
2008 return 1;
2010 mmap_unlock();
2011 return 0;
2013 #endif /* CONFIG_USER_ONLY */