tpm: Avoid qerror_report() outside QMP command handlers
[qemu.git] / translate-all.c
blobb6b0e1c098a96ff160f4917f2447576f7d754f5f
1 /*
2 * Host code generation
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
31 #include "config.h"
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "trace.h"
37 #include "disas/disas.h"
38 #include "tcg.h"
39 #if defined(CONFIG_USER_ONLY)
40 #include "qemu.h"
41 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
42 #include <sys/param.h>
43 #if __FreeBSD_version >= 700104
44 #define HAVE_KINFO_GETVMMAP
45 #define sigqueue sigqueue_freebsd /* avoid redefinition */
46 #include <sys/time.h>
47 #include <sys/proc.h>
48 #include <machine/profile.h>
49 #define _KERNEL
50 #include <sys/user.h>
51 #undef _KERNEL
52 #undef sigqueue
53 #include <libutil.h>
54 #endif
55 #endif
56 #else
57 #include "exec/address-spaces.h"
58 #endif
60 #include "exec/cputlb.h"
61 #include "translate-all.h"
62 #include "qemu/bitmap.h"
63 #include "qemu/timer.h"
65 //#define DEBUG_TB_INVALIDATE
66 //#define DEBUG_FLUSH
67 /* make various TB consistency checks */
68 //#define DEBUG_TB_CHECK
70 #if !defined(CONFIG_USER_ONLY)
71 /* TB consistency checks only implemented for usermode emulation. */
72 #undef DEBUG_TB_CHECK
73 #endif
75 #define SMC_BITMAP_USE_THRESHOLD 10
77 typedef struct PageDesc {
78 /* list of TBs intersecting this ram page */
79 TranslationBlock *first_tb;
80 /* in order to optimize self modifying code, we count the number
81 of lookups we do to a given page to use a bitmap */
82 unsigned int code_write_count;
83 unsigned long *code_bitmap;
84 #if defined(CONFIG_USER_ONLY)
85 unsigned long flags;
86 #endif
87 } PageDesc;
89 /* In system mode we want L1_MAP to be based on ram offsets,
90 while in user mode we want it to be based on virtual addresses. */
91 #if !defined(CONFIG_USER_ONLY)
92 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
93 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
94 #else
95 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
96 #endif
97 #else
98 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
99 #endif
101 /* Size of the L2 (and L3, etc) page tables. */
102 #define V_L2_BITS 10
103 #define V_L2_SIZE (1 << V_L2_BITS)
105 /* The bits remaining after N lower levels of page tables. */
106 #define V_L1_BITS_REM \
107 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
109 #if V_L1_BITS_REM < 4
110 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
111 #else
112 #define V_L1_BITS V_L1_BITS_REM
113 #endif
115 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
117 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
119 uintptr_t qemu_real_host_page_size;
120 uintptr_t qemu_host_page_size;
121 uintptr_t qemu_host_page_mask;
123 /* This is a multi-level map on the virtual address space.
124 The bottom level has pointers to PageDesc. */
125 static void *l1_map[V_L1_SIZE];
127 /* code generation context */
128 TCGContext tcg_ctx;
130 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
131 tb_page_addr_t phys_page2);
132 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
134 void cpu_gen_init(void)
136 tcg_context_init(&tcg_ctx);
139 /* return non zero if the very first instruction is invalid so that
140 the virtual CPU can trigger an exception.
142 '*gen_code_size_ptr' contains the size of the generated code (host
143 code).
145 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
147 TCGContext *s = &tcg_ctx;
148 tcg_insn_unit *gen_code_buf;
149 int gen_code_size;
150 #ifdef CONFIG_PROFILER
151 int64_t ti;
152 #endif
154 #ifdef CONFIG_PROFILER
155 s->tb_count1++; /* includes aborted translations because of
156 exceptions */
157 ti = profile_getclock();
158 #endif
159 tcg_func_start(s);
161 gen_intermediate_code(env, tb);
163 trace_translate_block(tb, tb->pc, tb->tc_ptr);
165 /* generate machine code */
166 gen_code_buf = tb->tc_ptr;
167 tb->tb_next_offset[0] = 0xffff;
168 tb->tb_next_offset[1] = 0xffff;
169 s->tb_next_offset = tb->tb_next_offset;
170 #ifdef USE_DIRECT_JUMP
171 s->tb_jmp_offset = tb->tb_jmp_offset;
172 s->tb_next = NULL;
173 #else
174 s->tb_jmp_offset = NULL;
175 s->tb_next = tb->tb_next;
176 #endif
178 #ifdef CONFIG_PROFILER
179 s->tb_count++;
180 s->interm_time += profile_getclock() - ti;
181 s->code_time -= profile_getclock();
182 #endif
183 gen_code_size = tcg_gen_code(s, gen_code_buf);
184 *gen_code_size_ptr = gen_code_size;
185 #ifdef CONFIG_PROFILER
186 s->code_time += profile_getclock();
187 s->code_in_len += tb->size;
188 s->code_out_len += gen_code_size;
189 #endif
191 #ifdef DEBUG_DISAS
192 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
193 qemu_log("OUT: [size=%d]\n", gen_code_size);
194 log_disas(tb->tc_ptr, gen_code_size);
195 qemu_log("\n");
196 qemu_log_flush();
198 #endif
199 return 0;
202 /* The cpu state corresponding to 'searched_pc' is restored.
204 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
205 uintptr_t searched_pc)
207 CPUArchState *env = cpu->env_ptr;
208 TCGContext *s = &tcg_ctx;
209 int j;
210 uintptr_t tc_ptr;
211 #ifdef CONFIG_PROFILER
212 int64_t ti;
213 #endif
215 #ifdef CONFIG_PROFILER
216 ti = profile_getclock();
217 #endif
218 tcg_func_start(s);
220 gen_intermediate_code_pc(env, tb);
222 if (tb->cflags & CF_USE_ICOUNT) {
223 /* Reset the cycle counter to the start of the block. */
224 cpu->icount_decr.u16.low += tb->icount;
225 /* Clear the IO flag. */
226 cpu->can_do_io = 0;
229 /* find opc index corresponding to search_pc */
230 tc_ptr = (uintptr_t)tb->tc_ptr;
231 if (searched_pc < tc_ptr)
232 return -1;
234 s->tb_next_offset = tb->tb_next_offset;
235 #ifdef USE_DIRECT_JUMP
236 s->tb_jmp_offset = tb->tb_jmp_offset;
237 s->tb_next = NULL;
238 #else
239 s->tb_jmp_offset = NULL;
240 s->tb_next = tb->tb_next;
241 #endif
242 j = tcg_gen_code_search_pc(s, (tcg_insn_unit *)tc_ptr,
243 searched_pc - tc_ptr);
244 if (j < 0)
245 return -1;
246 /* now find start of instruction before */
247 while (s->gen_opc_instr_start[j] == 0) {
248 j--;
250 cpu->icount_decr.u16.low -= s->gen_opc_icount[j];
252 restore_state_to_opc(env, tb, j);
254 #ifdef CONFIG_PROFILER
255 s->restore_time += profile_getclock() - ti;
256 s->restore_count++;
257 #endif
258 return 0;
261 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
263 TranslationBlock *tb;
265 tb = tb_find_pc(retaddr);
266 if (tb) {
267 cpu_restore_state_from_tb(cpu, tb, retaddr);
268 if (tb->cflags & CF_NOCACHE) {
269 /* one-shot translation, invalidate it immediately */
270 cpu->current_tb = NULL;
271 tb_phys_invalidate(tb, -1);
272 tb_free(tb);
274 return true;
276 return false;
279 #ifdef _WIN32
280 static __attribute__((unused)) void map_exec(void *addr, long size)
282 DWORD old_protect;
283 VirtualProtect(addr, size,
284 PAGE_EXECUTE_READWRITE, &old_protect);
286 #else
287 static __attribute__((unused)) void map_exec(void *addr, long size)
289 unsigned long start, end, page_size;
291 page_size = getpagesize();
292 start = (unsigned long)addr;
293 start &= ~(page_size - 1);
295 end = (unsigned long)addr + size;
296 end += page_size - 1;
297 end &= ~(page_size - 1);
299 mprotect((void *)start, end - start,
300 PROT_READ | PROT_WRITE | PROT_EXEC);
302 #endif
304 void page_size_init(void)
306 /* NOTE: we can always suppose that qemu_host_page_size >=
307 TARGET_PAGE_SIZE */
308 qemu_real_host_page_size = getpagesize();
309 if (qemu_host_page_size == 0) {
310 qemu_host_page_size = qemu_real_host_page_size;
312 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
313 qemu_host_page_size = TARGET_PAGE_SIZE;
315 qemu_host_page_mask = ~(qemu_host_page_size - 1);
318 static void page_init(void)
320 page_size_init();
321 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
323 #ifdef HAVE_KINFO_GETVMMAP
324 struct kinfo_vmentry *freep;
325 int i, cnt;
327 freep = kinfo_getvmmap(getpid(), &cnt);
328 if (freep) {
329 mmap_lock();
330 for (i = 0; i < cnt; i++) {
331 unsigned long startaddr, endaddr;
333 startaddr = freep[i].kve_start;
334 endaddr = freep[i].kve_end;
335 if (h2g_valid(startaddr)) {
336 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
338 if (h2g_valid(endaddr)) {
339 endaddr = h2g(endaddr);
340 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
341 } else {
342 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
343 endaddr = ~0ul;
344 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
345 #endif
349 free(freep);
350 mmap_unlock();
352 #else
353 FILE *f;
355 last_brk = (unsigned long)sbrk(0);
357 f = fopen("/compat/linux/proc/self/maps", "r");
358 if (f) {
359 mmap_lock();
361 do {
362 unsigned long startaddr, endaddr;
363 int n;
365 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
367 if (n == 2 && h2g_valid(startaddr)) {
368 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
370 if (h2g_valid(endaddr)) {
371 endaddr = h2g(endaddr);
372 } else {
373 endaddr = ~0ul;
375 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
377 } while (!feof(f));
379 fclose(f);
380 mmap_unlock();
382 #endif
384 #endif
387 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
389 PageDesc *pd;
390 void **lp;
391 int i;
393 /* Level 1. Always allocated. */
394 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
396 /* Level 2..N-1. */
397 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
398 void **p = *lp;
400 if (p == NULL) {
401 if (!alloc) {
402 return NULL;
404 p = g_new0(void *, V_L2_SIZE);
405 *lp = p;
408 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
411 pd = *lp;
412 if (pd == NULL) {
413 if (!alloc) {
414 return NULL;
416 pd = g_new0(PageDesc, V_L2_SIZE);
417 *lp = pd;
420 return pd + (index & (V_L2_SIZE - 1));
423 static inline PageDesc *page_find(tb_page_addr_t index)
425 return page_find_alloc(index, 0);
428 #if !defined(CONFIG_USER_ONLY)
429 #define mmap_lock() do { } while (0)
430 #define mmap_unlock() do { } while (0)
431 #endif
433 #if defined(CONFIG_USER_ONLY)
434 /* Currently it is not recommended to allocate big chunks of data in
435 user mode. It will change when a dedicated libc will be used. */
436 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
437 region in which the guest needs to run. Revisit this. */
438 #define USE_STATIC_CODE_GEN_BUFFER
439 #endif
441 /* ??? Should configure for this, not list operating systems here. */
442 #if (defined(__linux__) \
443 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
444 || defined(__DragonFly__) || defined(__OpenBSD__) \
445 || defined(__NetBSD__))
446 # define USE_MMAP
447 #endif
449 /* Minimum size of the code gen buffer. This number is randomly chosen,
450 but not so small that we can't have a fair number of TB's live. */
451 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
453 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
454 indicated, this is constrained by the range of direct branches on the
455 host cpu, as used by the TCG implementation of goto_tb. */
456 #if defined(__x86_64__)
457 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
458 #elif defined(__sparc__)
459 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
460 #elif defined(__aarch64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
462 #elif defined(__arm__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
464 #elif defined(__s390x__)
465 /* We have a +- 4GB range on the branches; leave some slop. */
466 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
467 #elif defined(__mips__)
468 /* We have a 256MB branch region, but leave room to make sure the
469 main executable is also within that region. */
470 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
471 #else
472 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
473 #endif
475 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
477 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
478 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
479 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
481 static inline size_t size_code_gen_buffer(size_t tb_size)
483 /* Size the buffer. */
484 if (tb_size == 0) {
485 #ifdef USE_STATIC_CODE_GEN_BUFFER
486 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
487 #else
488 /* ??? Needs adjustments. */
489 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
490 static buffer, we could size this on RESERVED_VA, on the text
491 segment size of the executable, or continue to use the default. */
492 tb_size = (unsigned long)(ram_size / 4);
493 #endif
495 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
496 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
498 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
499 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
501 tcg_ctx.code_gen_buffer_size = tb_size;
502 return tb_size;
505 #ifdef __mips__
506 /* In order to use J and JAL within the code_gen_buffer, we require
507 that the buffer not cross a 256MB boundary. */
508 static inline bool cross_256mb(void *addr, size_t size)
510 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & 0xf0000000;
513 /* We weren't able to allocate a buffer without crossing that boundary,
514 so make do with the larger portion of the buffer that doesn't cross.
515 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
516 static inline void *split_cross_256mb(void *buf1, size_t size1)
518 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & 0xf0000000);
519 size_t size2 = buf1 + size1 - buf2;
521 size1 = buf2 - buf1;
522 if (size1 < size2) {
523 size1 = size2;
524 buf1 = buf2;
527 tcg_ctx.code_gen_buffer_size = size1;
528 return buf1;
530 #endif
532 #ifdef USE_STATIC_CODE_GEN_BUFFER
533 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
534 __attribute__((aligned(CODE_GEN_ALIGN)));
536 static inline void *alloc_code_gen_buffer(void)
538 void *buf = static_code_gen_buffer;
539 #ifdef __mips__
540 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
541 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
543 #endif
544 map_exec(buf, tcg_ctx.code_gen_buffer_size);
545 return buf;
547 #elif defined(USE_MMAP)
548 static inline void *alloc_code_gen_buffer(void)
550 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
551 uintptr_t start = 0;
552 void *buf;
554 /* Constrain the position of the buffer based on the host cpu.
555 Note that these addresses are chosen in concert with the
556 addresses assigned in the relevant linker script file. */
557 # if defined(__PIE__) || defined(__PIC__)
558 /* Don't bother setting a preferred location if we're building
559 a position-independent executable. We're more likely to get
560 an address near the main executable if we let the kernel
561 choose the address. */
562 # elif defined(__x86_64__) && defined(MAP_32BIT)
563 /* Force the memory down into low memory with the executable.
564 Leave the choice of exact location with the kernel. */
565 flags |= MAP_32BIT;
566 /* Cannot expect to map more than 800MB in low memory. */
567 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
568 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
570 # elif defined(__sparc__)
571 start = 0x40000000ul;
572 # elif defined(__s390x__)
573 start = 0x90000000ul;
574 # elif defined(__mips__)
575 /* ??? We ought to more explicitly manage layout for softmmu too. */
576 # ifdef CONFIG_USER_ONLY
577 start = 0x68000000ul;
578 # elif _MIPS_SIM == _ABI64
579 start = 0x128000000ul;
580 # else
581 start = 0x08000000ul;
582 # endif
583 # endif
585 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
586 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
587 if (buf == MAP_FAILED) {
588 return NULL;
591 #ifdef __mips__
592 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
593 /* Try again, with the original still mapped, to avoid re-acquiring
594 that 256mb crossing. This time don't specify an address. */
595 size_t size2, size1 = tcg_ctx.code_gen_buffer_size;
596 void *buf2 = mmap(NULL, size1, PROT_WRITE | PROT_READ | PROT_EXEC,
597 flags, -1, 0);
598 if (buf2 != MAP_FAILED) {
599 if (!cross_256mb(buf2, size1)) {
600 /* Success! Use the new buffer. */
601 munmap(buf, size1);
602 return buf2;
604 /* Failure. Work with what we had. */
605 munmap(buf2, size1);
608 /* Split the original buffer. Free the smaller half. */
609 buf2 = split_cross_256mb(buf, size1);
610 size2 = tcg_ctx.code_gen_buffer_size;
611 munmap(buf + (buf == buf2 ? size2 : 0), size1 - size2);
612 return buf2;
614 #endif
616 return buf;
618 #else
619 static inline void *alloc_code_gen_buffer(void)
621 void *buf = g_try_malloc(tcg_ctx.code_gen_buffer_size);
623 if (buf == NULL) {
624 return NULL;
627 #ifdef __mips__
628 if (cross_256mb(buf, tcg_ctx.code_gen_buffer_size)) {
629 void *buf2 = g_malloc(tcg_ctx.code_gen_buffer_size);
630 if (buf2 != NULL && !cross_256mb(buf2, size1)) {
631 /* Success! Use the new buffer. */
632 free(buf);
633 buf = buf2;
634 } else {
635 /* Failure. Work with what we had. Since this is malloc
636 and not mmap, we can't free the other half. */
637 free(buf2);
638 buf = split_cross_256mb(buf, tcg_ctx.code_gen_buffer_size);
641 #endif
643 map_exec(buf, tcg_ctx.code_gen_buffer_size);
644 return buf;
646 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
648 static inline void code_gen_alloc(size_t tb_size)
650 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
651 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
652 if (tcg_ctx.code_gen_buffer == NULL) {
653 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
654 exit(1);
657 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
658 QEMU_MADV_HUGEPAGE);
660 /* Steal room for the prologue at the end of the buffer. This ensures
661 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
662 from TB's to the prologue are going to be in range. It also means
663 that we don't need to mark (additional) portions of the data segment
664 as executable. */
665 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
666 tcg_ctx.code_gen_buffer_size - 1024;
667 tcg_ctx.code_gen_buffer_size -= 1024;
669 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
670 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
671 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
672 CODE_GEN_AVG_BLOCK_SIZE;
673 tcg_ctx.tb_ctx.tbs =
674 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
677 /* Must be called before using the QEMU cpus. 'tb_size' is the size
678 (in bytes) allocated to the translation buffer. Zero means default
679 size. */
680 void tcg_exec_init(unsigned long tb_size)
682 cpu_gen_init();
683 code_gen_alloc(tb_size);
684 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
685 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
686 page_init();
687 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
688 /* There's no guest base to take into account, so go ahead and
689 initialize the prologue now. */
690 tcg_prologue_init(&tcg_ctx);
691 #endif
694 bool tcg_enabled(void)
696 return tcg_ctx.code_gen_buffer != NULL;
699 /* Allocate a new translation block. Flush the translation buffer if
700 too many translation blocks or too much generated code. */
701 static TranslationBlock *tb_alloc(target_ulong pc)
703 TranslationBlock *tb;
705 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
706 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
707 tcg_ctx.code_gen_buffer_max_size) {
708 return NULL;
710 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
711 tb->pc = pc;
712 tb->cflags = 0;
713 return tb;
716 void tb_free(TranslationBlock *tb)
718 /* In practice this is mostly used for single use temporary TB
719 Ignore the hard cases and just back up if this TB happens to
720 be the last one generated. */
721 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
722 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
723 tcg_ctx.code_gen_ptr = tb->tc_ptr;
724 tcg_ctx.tb_ctx.nb_tbs--;
728 static inline void invalidate_page_bitmap(PageDesc *p)
730 if (p->code_bitmap) {
731 g_free(p->code_bitmap);
732 p->code_bitmap = NULL;
734 p->code_write_count = 0;
737 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
738 static void page_flush_tb_1(int level, void **lp)
740 int i;
742 if (*lp == NULL) {
743 return;
745 if (level == 0) {
746 PageDesc *pd = *lp;
748 for (i = 0; i < V_L2_SIZE; ++i) {
749 pd[i].first_tb = NULL;
750 invalidate_page_bitmap(pd + i);
752 } else {
753 void **pp = *lp;
755 for (i = 0; i < V_L2_SIZE; ++i) {
756 page_flush_tb_1(level - 1, pp + i);
761 static void page_flush_tb(void)
763 int i;
765 for (i = 0; i < V_L1_SIZE; i++) {
766 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
770 /* flush all the translation blocks */
771 /* XXX: tb_flush is currently not thread safe */
772 void tb_flush(CPUArchState *env1)
774 CPUState *cpu = ENV_GET_CPU(env1);
776 #if defined(DEBUG_FLUSH)
777 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
778 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
779 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
780 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
781 tcg_ctx.tb_ctx.nb_tbs : 0);
782 #endif
783 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
784 > tcg_ctx.code_gen_buffer_size) {
785 cpu_abort(cpu, "Internal error: code buffer overflow\n");
787 tcg_ctx.tb_ctx.nb_tbs = 0;
789 CPU_FOREACH(cpu) {
790 memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
793 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0, sizeof(tcg_ctx.tb_ctx.tb_phys_hash));
794 page_flush_tb();
796 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
797 /* XXX: flush processor icache at this point if cache flush is
798 expensive */
799 tcg_ctx.tb_ctx.tb_flush_count++;
802 #ifdef DEBUG_TB_CHECK
804 static void tb_invalidate_check(target_ulong address)
806 TranslationBlock *tb;
807 int i;
809 address &= TARGET_PAGE_MASK;
810 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
811 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
812 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
813 address >= tb->pc + tb->size)) {
814 printf("ERROR invalidate: address=" TARGET_FMT_lx
815 " PC=%08lx size=%04x\n",
816 address, (long)tb->pc, tb->size);
822 /* verify that all the pages have correct rights for code */
823 static void tb_page_check(void)
825 TranslationBlock *tb;
826 int i, flags1, flags2;
828 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
829 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
830 tb = tb->phys_hash_next) {
831 flags1 = page_get_flags(tb->pc);
832 flags2 = page_get_flags(tb->pc + tb->size - 1);
833 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
834 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
835 (long)tb->pc, tb->size, flags1, flags2);
841 #endif
843 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
845 TranslationBlock *tb1;
847 for (;;) {
848 tb1 = *ptb;
849 if (tb1 == tb) {
850 *ptb = tb1->phys_hash_next;
851 break;
853 ptb = &tb1->phys_hash_next;
857 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
859 TranslationBlock *tb1;
860 unsigned int n1;
862 for (;;) {
863 tb1 = *ptb;
864 n1 = (uintptr_t)tb1 & 3;
865 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
866 if (tb1 == tb) {
867 *ptb = tb1->page_next[n1];
868 break;
870 ptb = &tb1->page_next[n1];
874 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
876 TranslationBlock *tb1, **ptb;
877 unsigned int n1;
879 ptb = &tb->jmp_next[n];
880 tb1 = *ptb;
881 if (tb1) {
882 /* find tb(n) in circular list */
883 for (;;) {
884 tb1 = *ptb;
885 n1 = (uintptr_t)tb1 & 3;
886 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
887 if (n1 == n && tb1 == tb) {
888 break;
890 if (n1 == 2) {
891 ptb = &tb1->jmp_first;
892 } else {
893 ptb = &tb1->jmp_next[n1];
896 /* now we can suppress tb(n) from the list */
897 *ptb = tb->jmp_next[n];
899 tb->jmp_next[n] = NULL;
903 /* reset the jump entry 'n' of a TB so that it is not chained to
904 another TB */
905 static inline void tb_reset_jump(TranslationBlock *tb, int n)
907 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
910 /* invalidate one TB */
911 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
913 CPUState *cpu;
914 PageDesc *p;
915 unsigned int h, n1;
916 tb_page_addr_t phys_pc;
917 TranslationBlock *tb1, *tb2;
919 /* remove the TB from the hash list */
920 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
921 h = tb_phys_hash_func(phys_pc);
922 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
924 /* remove the TB from the page list */
925 if (tb->page_addr[0] != page_addr) {
926 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
927 tb_page_remove(&p->first_tb, tb);
928 invalidate_page_bitmap(p);
930 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
931 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
932 tb_page_remove(&p->first_tb, tb);
933 invalidate_page_bitmap(p);
936 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
938 /* remove the TB from the hash list */
939 h = tb_jmp_cache_hash_func(tb->pc);
940 CPU_FOREACH(cpu) {
941 if (cpu->tb_jmp_cache[h] == tb) {
942 cpu->tb_jmp_cache[h] = NULL;
946 /* suppress this TB from the two jump lists */
947 tb_jmp_remove(tb, 0);
948 tb_jmp_remove(tb, 1);
950 /* suppress any remaining jumps to this TB */
951 tb1 = tb->jmp_first;
952 for (;;) {
953 n1 = (uintptr_t)tb1 & 3;
954 if (n1 == 2) {
955 break;
957 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
958 tb2 = tb1->jmp_next[n1];
959 tb_reset_jump(tb1, n1);
960 tb1->jmp_next[n1] = NULL;
961 tb1 = tb2;
963 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
965 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
968 static void build_page_bitmap(PageDesc *p)
970 int n, tb_start, tb_end;
971 TranslationBlock *tb;
973 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
975 tb = p->first_tb;
976 while (tb != NULL) {
977 n = (uintptr_t)tb & 3;
978 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
979 /* NOTE: this is subtle as a TB may span two physical pages */
980 if (n == 0) {
981 /* NOTE: tb_end may be after the end of the page, but
982 it is not a problem */
983 tb_start = tb->pc & ~TARGET_PAGE_MASK;
984 tb_end = tb_start + tb->size;
985 if (tb_end > TARGET_PAGE_SIZE) {
986 tb_end = TARGET_PAGE_SIZE;
988 } else {
989 tb_start = 0;
990 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
992 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
993 tb = tb->page_next[n];
997 TranslationBlock *tb_gen_code(CPUState *cpu,
998 target_ulong pc, target_ulong cs_base,
999 int flags, int cflags)
1001 CPUArchState *env = cpu->env_ptr;
1002 TranslationBlock *tb;
1003 tb_page_addr_t phys_pc, phys_page2;
1004 target_ulong virt_page2;
1005 int code_gen_size;
1007 phys_pc = get_page_addr_code(env, pc);
1008 if (use_icount) {
1009 cflags |= CF_USE_ICOUNT;
1011 tb = tb_alloc(pc);
1012 if (!tb) {
1013 /* flush must be done */
1014 tb_flush(env);
1015 /* cannot fail at this point */
1016 tb = tb_alloc(pc);
1017 /* Don't forget to invalidate previous TB info. */
1018 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
1020 tb->tc_ptr = tcg_ctx.code_gen_ptr;
1021 tb->cs_base = cs_base;
1022 tb->flags = flags;
1023 tb->cflags = cflags;
1024 cpu_gen_code(env, tb, &code_gen_size);
1025 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
1026 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
1028 /* check next page if needed */
1029 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1030 phys_page2 = -1;
1031 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1032 phys_page2 = get_page_addr_code(env, virt_page2);
1034 tb_link_page(tb, phys_pc, phys_page2);
1035 return tb;
1039 * Invalidate all TBs which intersect with the target physical address range
1040 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1041 * 'is_cpu_write_access' should be true if called from a real cpu write
1042 * access: the virtual CPU will exit the current TB if code is modified inside
1043 * this TB.
1045 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1047 while (start < end) {
1048 tb_invalidate_phys_page_range(start, end, 0);
1049 start &= TARGET_PAGE_MASK;
1050 start += TARGET_PAGE_SIZE;
1055 * Invalidate all TBs which intersect with the target physical address range
1056 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1057 * 'is_cpu_write_access' should be true if called from a real cpu write
1058 * access: the virtual CPU will exit the current TB if code is modified inside
1059 * this TB.
1061 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1062 int is_cpu_write_access)
1064 TranslationBlock *tb, *tb_next, *saved_tb;
1065 CPUState *cpu = current_cpu;
1066 #if defined(TARGET_HAS_PRECISE_SMC)
1067 CPUArchState *env = NULL;
1068 #endif
1069 tb_page_addr_t tb_start, tb_end;
1070 PageDesc *p;
1071 int n;
1072 #ifdef TARGET_HAS_PRECISE_SMC
1073 int current_tb_not_found = is_cpu_write_access;
1074 TranslationBlock *current_tb = NULL;
1075 int current_tb_modified = 0;
1076 target_ulong current_pc = 0;
1077 target_ulong current_cs_base = 0;
1078 int current_flags = 0;
1079 #endif /* TARGET_HAS_PRECISE_SMC */
1081 p = page_find(start >> TARGET_PAGE_BITS);
1082 if (!p) {
1083 return;
1085 #if defined(TARGET_HAS_PRECISE_SMC)
1086 if (cpu != NULL) {
1087 env = cpu->env_ptr;
1089 #endif
1091 /* we remove all the TBs in the range [start, end[ */
1092 /* XXX: see if in some cases it could be faster to invalidate all
1093 the code */
1094 tb = p->first_tb;
1095 while (tb != NULL) {
1096 n = (uintptr_t)tb & 3;
1097 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1098 tb_next = tb->page_next[n];
1099 /* NOTE: this is subtle as a TB may span two physical pages */
1100 if (n == 0) {
1101 /* NOTE: tb_end may be after the end of the page, but
1102 it is not a problem */
1103 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1104 tb_end = tb_start + tb->size;
1105 } else {
1106 tb_start = tb->page_addr[1];
1107 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1109 if (!(tb_end <= start || tb_start >= end)) {
1110 #ifdef TARGET_HAS_PRECISE_SMC
1111 if (current_tb_not_found) {
1112 current_tb_not_found = 0;
1113 current_tb = NULL;
1114 if (cpu->mem_io_pc) {
1115 /* now we have a real cpu fault */
1116 current_tb = tb_find_pc(cpu->mem_io_pc);
1119 if (current_tb == tb &&
1120 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1121 /* If we are modifying the current TB, we must stop
1122 its execution. We could be more precise by checking
1123 that the modification is after the current PC, but it
1124 would require a specialized function to partially
1125 restore the CPU state */
1127 current_tb_modified = 1;
1128 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1129 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1130 &current_flags);
1132 #endif /* TARGET_HAS_PRECISE_SMC */
1133 /* we need to do that to handle the case where a signal
1134 occurs while doing tb_phys_invalidate() */
1135 saved_tb = NULL;
1136 if (cpu != NULL) {
1137 saved_tb = cpu->current_tb;
1138 cpu->current_tb = NULL;
1140 tb_phys_invalidate(tb, -1);
1141 if (cpu != NULL) {
1142 cpu->current_tb = saved_tb;
1143 if (cpu->interrupt_request && cpu->current_tb) {
1144 cpu_interrupt(cpu, cpu->interrupt_request);
1148 tb = tb_next;
1150 #if !defined(CONFIG_USER_ONLY)
1151 /* if no code remaining, no need to continue to use slow writes */
1152 if (!p->first_tb) {
1153 invalidate_page_bitmap(p);
1154 tlb_unprotect_code(start);
1156 #endif
1157 #ifdef TARGET_HAS_PRECISE_SMC
1158 if (current_tb_modified) {
1159 /* we generate a block containing just the instruction
1160 modifying the memory. It will ensure that it cannot modify
1161 itself */
1162 cpu->current_tb = NULL;
1163 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1164 cpu_resume_from_signal(cpu, NULL);
1166 #endif
1169 /* len must be <= 8 and start must be a multiple of len */
1170 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1172 PageDesc *p;
1174 #if 0
1175 if (1) {
1176 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1177 cpu_single_env->mem_io_vaddr, len,
1178 cpu_single_env->eip,
1179 cpu_single_env->eip +
1180 (intptr_t)cpu_single_env->segs[R_CS].base);
1182 #endif
1183 p = page_find(start >> TARGET_PAGE_BITS);
1184 if (!p) {
1185 return;
1187 if (!p->code_bitmap &&
1188 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1189 /* build code bitmap */
1190 build_page_bitmap(p);
1192 if (p->code_bitmap) {
1193 unsigned int nr;
1194 unsigned long b;
1196 nr = start & ~TARGET_PAGE_MASK;
1197 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1198 if (b & ((1 << len) - 1)) {
1199 goto do_invalidate;
1201 } else {
1202 do_invalidate:
1203 tb_invalidate_phys_page_range(start, start + len, 1);
1207 #if !defined(CONFIG_SOFTMMU)
1208 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1209 uintptr_t pc, void *puc,
1210 bool locked)
1212 TranslationBlock *tb;
1213 PageDesc *p;
1214 int n;
1215 #ifdef TARGET_HAS_PRECISE_SMC
1216 TranslationBlock *current_tb = NULL;
1217 CPUState *cpu = current_cpu;
1218 CPUArchState *env = NULL;
1219 int current_tb_modified = 0;
1220 target_ulong current_pc = 0;
1221 target_ulong current_cs_base = 0;
1222 int current_flags = 0;
1223 #endif
1225 addr &= TARGET_PAGE_MASK;
1226 p = page_find(addr >> TARGET_PAGE_BITS);
1227 if (!p) {
1228 return;
1230 tb = p->first_tb;
1231 #ifdef TARGET_HAS_PRECISE_SMC
1232 if (tb && pc != 0) {
1233 current_tb = tb_find_pc(pc);
1235 if (cpu != NULL) {
1236 env = cpu->env_ptr;
1238 #endif
1239 while (tb != NULL) {
1240 n = (uintptr_t)tb & 3;
1241 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1242 #ifdef TARGET_HAS_PRECISE_SMC
1243 if (current_tb == tb &&
1244 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1245 /* If we are modifying the current TB, we must stop
1246 its execution. We could be more precise by checking
1247 that the modification is after the current PC, but it
1248 would require a specialized function to partially
1249 restore the CPU state */
1251 current_tb_modified = 1;
1252 cpu_restore_state_from_tb(cpu, current_tb, pc);
1253 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1254 &current_flags);
1256 #endif /* TARGET_HAS_PRECISE_SMC */
1257 tb_phys_invalidate(tb, addr);
1258 tb = tb->page_next[n];
1260 p->first_tb = NULL;
1261 #ifdef TARGET_HAS_PRECISE_SMC
1262 if (current_tb_modified) {
1263 /* we generate a block containing just the instruction
1264 modifying the memory. It will ensure that it cannot modify
1265 itself */
1266 cpu->current_tb = NULL;
1267 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1268 if (locked) {
1269 mmap_unlock();
1271 cpu_resume_from_signal(cpu, puc);
1273 #endif
1275 #endif
1277 /* add the tb in the target page and protect it if necessary */
1278 static inline void tb_alloc_page(TranslationBlock *tb,
1279 unsigned int n, tb_page_addr_t page_addr)
1281 PageDesc *p;
1282 #ifndef CONFIG_USER_ONLY
1283 bool page_already_protected;
1284 #endif
1286 tb->page_addr[n] = page_addr;
1287 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1288 tb->page_next[n] = p->first_tb;
1289 #ifndef CONFIG_USER_ONLY
1290 page_already_protected = p->first_tb != NULL;
1291 #endif
1292 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1293 invalidate_page_bitmap(p);
1295 #if defined(CONFIG_USER_ONLY)
1296 if (p->flags & PAGE_WRITE) {
1297 target_ulong addr;
1298 PageDesc *p2;
1299 int prot;
1301 /* force the host page as non writable (writes will have a
1302 page fault + mprotect overhead) */
1303 page_addr &= qemu_host_page_mask;
1304 prot = 0;
1305 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1306 addr += TARGET_PAGE_SIZE) {
1308 p2 = page_find(addr >> TARGET_PAGE_BITS);
1309 if (!p2) {
1310 continue;
1312 prot |= p2->flags;
1313 p2->flags &= ~PAGE_WRITE;
1315 mprotect(g2h(page_addr), qemu_host_page_size,
1316 (prot & PAGE_BITS) & ~PAGE_WRITE);
1317 #ifdef DEBUG_TB_INVALIDATE
1318 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1319 page_addr);
1320 #endif
1322 #else
1323 /* if some code is already present, then the pages are already
1324 protected. So we handle the case where only the first TB is
1325 allocated in a physical page */
1326 if (!page_already_protected) {
1327 tlb_protect_code(page_addr);
1329 #endif
1332 /* add a new TB and link it to the physical page tables. phys_page2 is
1333 (-1) to indicate that only one page contains the TB. */
1334 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1335 tb_page_addr_t phys_page2)
1337 unsigned int h;
1338 TranslationBlock **ptb;
1340 /* Grab the mmap lock to stop another thread invalidating this TB
1341 before we are done. */
1342 mmap_lock();
1343 /* add in the physical hash table */
1344 h = tb_phys_hash_func(phys_pc);
1345 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1346 tb->phys_hash_next = *ptb;
1347 *ptb = tb;
1349 /* add in the page list */
1350 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1351 if (phys_page2 != -1) {
1352 tb_alloc_page(tb, 1, phys_page2);
1353 } else {
1354 tb->page_addr[1] = -1;
1357 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1358 tb->jmp_next[0] = NULL;
1359 tb->jmp_next[1] = NULL;
1361 /* init original jump addresses */
1362 if (tb->tb_next_offset[0] != 0xffff) {
1363 tb_reset_jump(tb, 0);
1365 if (tb->tb_next_offset[1] != 0xffff) {
1366 tb_reset_jump(tb, 1);
1369 #ifdef DEBUG_TB_CHECK
1370 tb_page_check();
1371 #endif
1372 mmap_unlock();
1375 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1376 tb[1].tc_ptr. Return NULL if not found */
1377 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1379 int m_min, m_max, m;
1380 uintptr_t v;
1381 TranslationBlock *tb;
1383 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1384 return NULL;
1386 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1387 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1388 return NULL;
1390 /* binary search (cf Knuth) */
1391 m_min = 0;
1392 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1393 while (m_min <= m_max) {
1394 m = (m_min + m_max) >> 1;
1395 tb = &tcg_ctx.tb_ctx.tbs[m];
1396 v = (uintptr_t)tb->tc_ptr;
1397 if (v == tc_ptr) {
1398 return tb;
1399 } else if (tc_ptr < v) {
1400 m_max = m - 1;
1401 } else {
1402 m_min = m + 1;
1405 return &tcg_ctx.tb_ctx.tbs[m_max];
1408 #if !defined(CONFIG_USER_ONLY)
1409 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1411 ram_addr_t ram_addr;
1412 MemoryRegion *mr;
1413 hwaddr l = 1;
1415 rcu_read_lock();
1416 mr = address_space_translate(as, addr, &addr, &l, false);
1417 if (!(memory_region_is_ram(mr)
1418 || memory_region_is_romd(mr))) {
1419 rcu_read_unlock();
1420 return;
1422 ram_addr = (memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK)
1423 + addr;
1424 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1425 rcu_read_unlock();
1427 #endif /* !defined(CONFIG_USER_ONLY) */
1429 void tb_check_watchpoint(CPUState *cpu)
1431 TranslationBlock *tb;
1433 tb = tb_find_pc(cpu->mem_io_pc);
1434 if (tb) {
1435 /* We can use retranslation to find the PC. */
1436 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1437 tb_phys_invalidate(tb, -1);
1438 } else {
1439 /* The exception probably happened in a helper. The CPU state should
1440 have been saved before calling it. Fetch the PC from there. */
1441 CPUArchState *env = cpu->env_ptr;
1442 target_ulong pc, cs_base;
1443 tb_page_addr_t addr;
1444 int flags;
1446 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1447 addr = get_page_addr_code(env, pc);
1448 tb_invalidate_phys_range(addr, addr + 1);
1452 #ifndef CONFIG_USER_ONLY
1453 /* mask must never be zero, except for A20 change call */
1454 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1456 int old_mask;
1458 old_mask = cpu->interrupt_request;
1459 cpu->interrupt_request |= mask;
1462 * If called from iothread context, wake the target cpu in
1463 * case its halted.
1465 if (!qemu_cpu_is_self(cpu)) {
1466 qemu_cpu_kick(cpu);
1467 return;
1470 if (use_icount) {
1471 cpu->icount_decr.u16.high = 0xffff;
1472 if (!cpu_can_do_io(cpu)
1473 && (mask & ~old_mask) != 0) {
1474 cpu_abort(cpu, "Raised interrupt while not in I/O function");
1476 } else {
1477 cpu->tcg_exit_req = 1;
1481 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1483 /* in deterministic execution mode, instructions doing device I/Os
1484 must be at the end of the TB */
1485 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1487 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1488 CPUArchState *env = cpu->env_ptr;
1489 #endif
1490 TranslationBlock *tb;
1491 uint32_t n, cflags;
1492 target_ulong pc, cs_base;
1493 uint64_t flags;
1495 tb = tb_find_pc(retaddr);
1496 if (!tb) {
1497 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1498 (void *)retaddr);
1500 n = cpu->icount_decr.u16.low + tb->icount;
1501 cpu_restore_state_from_tb(cpu, tb, retaddr);
1502 /* Calculate how many instructions had been executed before the fault
1503 occurred. */
1504 n = n - cpu->icount_decr.u16.low;
1505 /* Generate a new TB ending on the I/O insn. */
1506 n++;
1507 /* On MIPS and SH, delay slot instructions can only be restarted if
1508 they were already the first instruction in the TB. If this is not
1509 the first instruction in a TB then re-execute the preceding
1510 branch. */
1511 #if defined(TARGET_MIPS)
1512 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1513 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1514 cpu->icount_decr.u16.low++;
1515 env->hflags &= ~MIPS_HFLAG_BMASK;
1517 #elif defined(TARGET_SH4)
1518 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1519 && n > 1) {
1520 env->pc -= 2;
1521 cpu->icount_decr.u16.low++;
1522 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1524 #endif
1525 /* This should never happen. */
1526 if (n > CF_COUNT_MASK) {
1527 cpu_abort(cpu, "TB too big during recompile");
1530 cflags = n | CF_LAST_IO;
1531 pc = tb->pc;
1532 cs_base = tb->cs_base;
1533 flags = tb->flags;
1534 tb_phys_invalidate(tb, -1);
1535 /* FIXME: In theory this could raise an exception. In practice
1536 we have already translated the block once so it's probably ok. */
1537 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1538 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1539 the first in the TB) then we end up generating a whole new TB and
1540 repeating the fault, which is horribly inefficient.
1541 Better would be to execute just this insn uncached, or generate a
1542 second new TB. */
1543 cpu_resume_from_signal(cpu, NULL);
1546 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1548 unsigned int i;
1550 /* Discard jump cache entries for any tb which might potentially
1551 overlap the flushed page. */
1552 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1553 memset(&cpu->tb_jmp_cache[i], 0,
1554 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1556 i = tb_jmp_cache_hash_page(addr);
1557 memset(&cpu->tb_jmp_cache[i], 0,
1558 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1561 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1563 int i, target_code_size, max_target_code_size;
1564 int direct_jmp_count, direct_jmp2_count, cross_page;
1565 TranslationBlock *tb;
1567 target_code_size = 0;
1568 max_target_code_size = 0;
1569 cross_page = 0;
1570 direct_jmp_count = 0;
1571 direct_jmp2_count = 0;
1572 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1573 tb = &tcg_ctx.tb_ctx.tbs[i];
1574 target_code_size += tb->size;
1575 if (tb->size > max_target_code_size) {
1576 max_target_code_size = tb->size;
1578 if (tb->page_addr[1] != -1) {
1579 cross_page++;
1581 if (tb->tb_next_offset[0] != 0xffff) {
1582 direct_jmp_count++;
1583 if (tb->tb_next_offset[1] != 0xffff) {
1584 direct_jmp2_count++;
1588 /* XXX: avoid using doubles ? */
1589 cpu_fprintf(f, "Translation buffer state:\n");
1590 cpu_fprintf(f, "gen code size %td/%zd\n",
1591 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1592 tcg_ctx.code_gen_buffer_max_size);
1593 cpu_fprintf(f, "TB count %d/%d\n",
1594 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1595 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1596 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1597 tcg_ctx.tb_ctx.nb_tbs : 0,
1598 max_target_code_size);
1599 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1600 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1601 tcg_ctx.code_gen_buffer) /
1602 tcg_ctx.tb_ctx.nb_tbs : 0,
1603 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1604 tcg_ctx.code_gen_buffer) /
1605 target_code_size : 0);
1606 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1607 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1608 tcg_ctx.tb_ctx.nb_tbs : 0);
1609 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1610 direct_jmp_count,
1611 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1612 tcg_ctx.tb_ctx.nb_tbs : 0,
1613 direct_jmp2_count,
1614 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1615 tcg_ctx.tb_ctx.nb_tbs : 0);
1616 cpu_fprintf(f, "\nStatistics:\n");
1617 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1618 cpu_fprintf(f, "TB invalidate count %d\n",
1619 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1620 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1621 tcg_dump_info(f, cpu_fprintf);
1624 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1626 tcg_dump_op_count(f, cpu_fprintf);
1629 #else /* CONFIG_USER_ONLY */
1631 void cpu_interrupt(CPUState *cpu, int mask)
1633 cpu->interrupt_request |= mask;
1634 cpu->tcg_exit_req = 1;
1638 * Walks guest process memory "regions" one by one
1639 * and calls callback function 'fn' for each region.
1641 struct walk_memory_regions_data {
1642 walk_memory_regions_fn fn;
1643 void *priv;
1644 target_ulong start;
1645 int prot;
1648 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1649 target_ulong end, int new_prot)
1651 if (data->start != -1u) {
1652 int rc = data->fn(data->priv, data->start, end, data->prot);
1653 if (rc != 0) {
1654 return rc;
1658 data->start = (new_prot ? end : -1u);
1659 data->prot = new_prot;
1661 return 0;
1664 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1665 target_ulong base, int level, void **lp)
1667 target_ulong pa;
1668 int i, rc;
1670 if (*lp == NULL) {
1671 return walk_memory_regions_end(data, base, 0);
1674 if (level == 0) {
1675 PageDesc *pd = *lp;
1677 for (i = 0; i < V_L2_SIZE; ++i) {
1678 int prot = pd[i].flags;
1680 pa = base | (i << TARGET_PAGE_BITS);
1681 if (prot != data->prot) {
1682 rc = walk_memory_regions_end(data, pa, prot);
1683 if (rc != 0) {
1684 return rc;
1688 } else {
1689 void **pp = *lp;
1691 for (i = 0; i < V_L2_SIZE; ++i) {
1692 pa = base | ((target_ulong)i <<
1693 (TARGET_PAGE_BITS + V_L2_BITS * level));
1694 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1695 if (rc != 0) {
1696 return rc;
1701 return 0;
1704 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1706 struct walk_memory_regions_data data;
1707 uintptr_t i;
1709 data.fn = fn;
1710 data.priv = priv;
1711 data.start = -1u;
1712 data.prot = 0;
1714 for (i = 0; i < V_L1_SIZE; i++) {
1715 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
1716 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1717 if (rc != 0) {
1718 return rc;
1722 return walk_memory_regions_end(&data, 0, 0);
1725 static int dump_region(void *priv, target_ulong start,
1726 target_ulong end, unsigned long prot)
1728 FILE *f = (FILE *)priv;
1730 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1731 " "TARGET_FMT_lx" %c%c%c\n",
1732 start, end, end - start,
1733 ((prot & PAGE_READ) ? 'r' : '-'),
1734 ((prot & PAGE_WRITE) ? 'w' : '-'),
1735 ((prot & PAGE_EXEC) ? 'x' : '-'));
1737 return 0;
1740 /* dump memory mappings */
1741 void page_dump(FILE *f)
1743 const int length = sizeof(target_ulong) * 2;
1744 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1745 length, "start", length, "end", length, "size", "prot");
1746 walk_memory_regions(f, dump_region);
1749 int page_get_flags(target_ulong address)
1751 PageDesc *p;
1753 p = page_find(address >> TARGET_PAGE_BITS);
1754 if (!p) {
1755 return 0;
1757 return p->flags;
1760 /* Modify the flags of a page and invalidate the code if necessary.
1761 The flag PAGE_WRITE_ORG is positioned automatically depending
1762 on PAGE_WRITE. The mmap_lock should already be held. */
1763 void page_set_flags(target_ulong start, target_ulong end, int flags)
1765 target_ulong addr, len;
1767 /* This function should never be called with addresses outside the
1768 guest address space. If this assert fires, it probably indicates
1769 a missing call to h2g_valid. */
1770 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1771 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1772 #endif
1773 assert(start < end);
1775 start = start & TARGET_PAGE_MASK;
1776 end = TARGET_PAGE_ALIGN(end);
1778 if (flags & PAGE_WRITE) {
1779 flags |= PAGE_WRITE_ORG;
1782 for (addr = start, len = end - start;
1783 len != 0;
1784 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1785 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1787 /* If the write protection bit is set, then we invalidate
1788 the code inside. */
1789 if (!(p->flags & PAGE_WRITE) &&
1790 (flags & PAGE_WRITE) &&
1791 p->first_tb) {
1792 tb_invalidate_phys_page(addr, 0, NULL, false);
1794 p->flags = flags;
1798 int page_check_range(target_ulong start, target_ulong len, int flags)
1800 PageDesc *p;
1801 target_ulong end;
1802 target_ulong addr;
1804 /* This function should never be called with addresses outside the
1805 guest address space. If this assert fires, it probably indicates
1806 a missing call to h2g_valid. */
1807 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1808 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1809 #endif
1811 if (len == 0) {
1812 return 0;
1814 if (start + len - 1 < start) {
1815 /* We've wrapped around. */
1816 return -1;
1819 /* must do before we loose bits in the next step */
1820 end = TARGET_PAGE_ALIGN(start + len);
1821 start = start & TARGET_PAGE_MASK;
1823 for (addr = start, len = end - start;
1824 len != 0;
1825 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1826 p = page_find(addr >> TARGET_PAGE_BITS);
1827 if (!p) {
1828 return -1;
1830 if (!(p->flags & PAGE_VALID)) {
1831 return -1;
1834 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1835 return -1;
1837 if (flags & PAGE_WRITE) {
1838 if (!(p->flags & PAGE_WRITE_ORG)) {
1839 return -1;
1841 /* unprotect the page if it was put read-only because it
1842 contains translated code */
1843 if (!(p->flags & PAGE_WRITE)) {
1844 if (!page_unprotect(addr, 0, NULL)) {
1845 return -1;
1850 return 0;
1853 /* called from signal handler: invalidate the code and unprotect the
1854 page. Return TRUE if the fault was successfully handled. */
1855 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1857 unsigned int prot;
1858 PageDesc *p;
1859 target_ulong host_start, host_end, addr;
1861 /* Technically this isn't safe inside a signal handler. However we
1862 know this only ever happens in a synchronous SEGV handler, so in
1863 practice it seems to be ok. */
1864 mmap_lock();
1866 p = page_find(address >> TARGET_PAGE_BITS);
1867 if (!p) {
1868 mmap_unlock();
1869 return 0;
1872 /* if the page was really writable, then we change its
1873 protection back to writable */
1874 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1875 host_start = address & qemu_host_page_mask;
1876 host_end = host_start + qemu_host_page_size;
1878 prot = 0;
1879 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1880 p = page_find(addr >> TARGET_PAGE_BITS);
1881 p->flags |= PAGE_WRITE;
1882 prot |= p->flags;
1884 /* and since the content will be modified, we must invalidate
1885 the corresponding translated code. */
1886 tb_invalidate_phys_page(addr, pc, puc, true);
1887 #ifdef DEBUG_TB_CHECK
1888 tb_invalidate_check(addr);
1889 #endif
1891 mprotect((void *)g2h(host_start), qemu_host_page_size,
1892 prot & PAGE_BITS);
1894 mmap_unlock();
1895 return 1;
1897 mmap_unlock();
1898 return 0;
1900 #endif /* CONFIG_USER_ONLY */