PPC: Fix rldcl
[qemu/agraf.git] / translate-all.c
blobda93608f0382a840154c2ba40712cae5e682c400
1 /*
2 * Host code generation
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #ifdef _WIN32
20 #include <windows.h>
21 #else
22 #include <sys/types.h>
23 #include <sys/mman.h>
24 #endif
25 #include <stdarg.h>
26 #include <stdlib.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <inttypes.h>
31 #include "config.h"
33 #include "qemu-common.h"
34 #define NO_CPU_IO_DEFS
35 #include "cpu.h"
36 #include "disas/disas.h"
37 #include "tcg.h"
38 #if defined(CONFIG_USER_ONLY)
39 #include "qemu.h"
40 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
41 #include <sys/param.h>
42 #if __FreeBSD_version >= 700104
43 #define HAVE_KINFO_GETVMMAP
44 #define sigqueue sigqueue_freebsd /* avoid redefinition */
45 #include <sys/time.h>
46 #include <sys/proc.h>
47 #include <machine/profile.h>
48 #define _KERNEL
49 #include <sys/user.h>
50 #undef _KERNEL
51 #undef sigqueue
52 #include <libutil.h>
53 #endif
54 #endif
55 #else
56 #include "exec/address-spaces.h"
57 #endif
58 #include "qemu/timer.h"
60 #include "exec/cputlb.h"
61 #include "translate-all.h"
62 #include "qemu/timer.h"
64 //#define DEBUG_TB_INVALIDATE
65 //#define DEBUG_FLUSH
66 /* make various TB consistency checks */
67 //#define DEBUG_TB_CHECK
69 #if !defined(CONFIG_USER_ONLY)
70 /* TB consistency checks only implemented for usermode emulation. */
71 #undef DEBUG_TB_CHECK
72 #endif
74 #define SMC_BITMAP_USE_THRESHOLD 10
76 typedef struct PageDesc {
77 /* list of TBs intersecting this ram page */
78 TranslationBlock *first_tb;
79 /* in order to optimize self modifying code, we count the number
80 of lookups we do to a given page to use a bitmap */
81 unsigned int code_write_count;
82 uint8_t *code_bitmap;
83 #if defined(CONFIG_USER_ONLY)
84 unsigned long flags;
85 #endif
86 } PageDesc;
88 /* In system mode we want L1_MAP to be based on ram offsets,
89 while in user mode we want it to be based on virtual addresses. */
90 #if !defined(CONFIG_USER_ONLY)
91 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
92 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
93 #else
94 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
95 #endif
96 #else
97 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
98 #endif
100 /* The bits remaining after N lower levels of page tables. */
101 #define V_L1_BITS_REM \
102 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
104 #if V_L1_BITS_REM < 4
105 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
106 #else
107 #define V_L1_BITS V_L1_BITS_REM
108 #endif
110 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
112 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
114 uintptr_t qemu_real_host_page_size;
115 uintptr_t qemu_host_page_size;
116 uintptr_t qemu_host_page_mask;
118 /* This is a multi-level map on the virtual address space.
119 The bottom level has pointers to PageDesc. */
120 static void *l1_map[V_L1_SIZE];
122 /* code generation context */
123 TCGContext tcg_ctx;
125 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
126 tb_page_addr_t phys_page2);
127 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
129 void cpu_gen_init(void)
131 tcg_context_init(&tcg_ctx);
134 /* return non zero if the very first instruction is invalid so that
135 the virtual CPU can trigger an exception.
137 '*gen_code_size_ptr' contains the size of the generated code (host
138 code).
140 int cpu_gen_code(CPUArchState *env, TranslationBlock *tb, int *gen_code_size_ptr)
142 TCGContext *s = &tcg_ctx;
143 uint8_t *gen_code_buf;
144 int gen_code_size;
145 #ifdef CONFIG_PROFILER
146 int64_t ti;
147 #endif
149 #ifdef CONFIG_PROFILER
150 s->tb_count1++; /* includes aborted translations because of
151 exceptions */
152 ti = profile_getclock();
153 #endif
154 tcg_func_start(s);
156 gen_intermediate_code(env, tb);
158 /* generate machine code */
159 gen_code_buf = tb->tc_ptr;
160 tb->tb_next_offset[0] = 0xffff;
161 tb->tb_next_offset[1] = 0xffff;
162 s->tb_next_offset = tb->tb_next_offset;
163 #ifdef USE_DIRECT_JUMP
164 s->tb_jmp_offset = tb->tb_jmp_offset;
165 s->tb_next = NULL;
166 #else
167 s->tb_jmp_offset = NULL;
168 s->tb_next = tb->tb_next;
169 #endif
171 #ifdef CONFIG_PROFILER
172 s->tb_count++;
173 s->interm_time += profile_getclock() - ti;
174 s->code_time -= profile_getclock();
175 #endif
176 gen_code_size = tcg_gen_code(s, gen_code_buf);
177 *gen_code_size_ptr = gen_code_size;
178 #ifdef CONFIG_PROFILER
179 s->code_time += profile_getclock();
180 s->code_in_len += tb->size;
181 s->code_out_len += gen_code_size;
182 #endif
184 #ifdef DEBUG_DISAS
185 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
186 qemu_log("OUT: [size=%d]\n", *gen_code_size_ptr);
187 log_disas(tb->tc_ptr, *gen_code_size_ptr);
188 qemu_log("\n");
189 qemu_log_flush();
191 #endif
192 return 0;
195 /* The cpu state corresponding to 'searched_pc' is restored.
197 static int cpu_restore_state_from_tb(TranslationBlock *tb, CPUArchState *env,
198 uintptr_t searched_pc)
200 TCGContext *s = &tcg_ctx;
201 int j;
202 uintptr_t tc_ptr;
203 #ifdef CONFIG_PROFILER
204 int64_t ti;
205 #endif
207 #ifdef CONFIG_PROFILER
208 ti = profile_getclock();
209 #endif
210 tcg_func_start(s);
212 gen_intermediate_code_pc(env, tb);
214 if (use_icount) {
215 /* Reset the cycle counter to the start of the block. */
216 env->icount_decr.u16.low += tb->icount;
217 /* Clear the IO flag. */
218 env->can_do_io = 0;
221 /* find opc index corresponding to search_pc */
222 tc_ptr = (uintptr_t)tb->tc_ptr;
223 if (searched_pc < tc_ptr)
224 return -1;
226 s->tb_next_offset = tb->tb_next_offset;
227 #ifdef USE_DIRECT_JUMP
228 s->tb_jmp_offset = tb->tb_jmp_offset;
229 s->tb_next = NULL;
230 #else
231 s->tb_jmp_offset = NULL;
232 s->tb_next = tb->tb_next;
233 #endif
234 j = tcg_gen_code_search_pc(s, (uint8_t *)tc_ptr, searched_pc - tc_ptr);
235 if (j < 0)
236 return -1;
237 /* now find start of instruction before */
238 while (s->gen_opc_instr_start[j] == 0) {
239 j--;
241 env->icount_decr.u16.low -= s->gen_opc_icount[j];
243 restore_state_to_opc(env, tb, j);
245 #ifdef CONFIG_PROFILER
246 s->restore_time += profile_getclock() - ti;
247 s->restore_count++;
248 #endif
249 return 0;
252 bool cpu_restore_state(CPUArchState *env, uintptr_t retaddr)
254 TranslationBlock *tb;
256 tb = tb_find_pc(retaddr);
257 if (tb) {
258 cpu_restore_state_from_tb(tb, env, retaddr);
259 return true;
261 return false;
264 #ifdef _WIN32
265 static inline void map_exec(void *addr, long size)
267 DWORD old_protect;
268 VirtualProtect(addr, size,
269 PAGE_EXECUTE_READWRITE, &old_protect);
271 #else
272 static inline void map_exec(void *addr, long size)
274 unsigned long start, end, page_size;
276 page_size = getpagesize();
277 start = (unsigned long)addr;
278 start &= ~(page_size - 1);
280 end = (unsigned long)addr + size;
281 end += page_size - 1;
282 end &= ~(page_size - 1);
284 mprotect((void *)start, end - start,
285 PROT_READ | PROT_WRITE | PROT_EXEC);
287 #endif
289 static void page_init(void)
291 /* NOTE: we can always suppose that qemu_host_page_size >=
292 TARGET_PAGE_SIZE */
293 #ifdef _WIN32
295 SYSTEM_INFO system_info;
297 GetSystemInfo(&system_info);
298 qemu_real_host_page_size = system_info.dwPageSize;
300 #else
301 qemu_real_host_page_size = getpagesize();
302 #endif
303 if (qemu_host_page_size == 0) {
304 qemu_host_page_size = qemu_real_host_page_size;
306 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
307 qemu_host_page_size = TARGET_PAGE_SIZE;
309 qemu_host_page_mask = ~(qemu_host_page_size - 1);
311 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
313 #ifdef HAVE_KINFO_GETVMMAP
314 struct kinfo_vmentry *freep;
315 int i, cnt;
317 freep = kinfo_getvmmap(getpid(), &cnt);
318 if (freep) {
319 mmap_lock();
320 for (i = 0; i < cnt; i++) {
321 unsigned long startaddr, endaddr;
323 startaddr = freep[i].kve_start;
324 endaddr = freep[i].kve_end;
325 if (h2g_valid(startaddr)) {
326 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
328 if (h2g_valid(endaddr)) {
329 endaddr = h2g(endaddr);
330 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
331 } else {
332 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
333 endaddr = ~0ul;
334 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
335 #endif
339 free(freep);
340 mmap_unlock();
342 #else
343 FILE *f;
345 last_brk = (unsigned long)sbrk(0);
347 f = fopen("/compat/linux/proc/self/maps", "r");
348 if (f) {
349 mmap_lock();
351 do {
352 unsigned long startaddr, endaddr;
353 int n;
355 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
357 if (n == 2 && h2g_valid(startaddr)) {
358 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
360 if (h2g_valid(endaddr)) {
361 endaddr = h2g(endaddr);
362 } else {
363 endaddr = ~0ul;
365 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
367 } while (!feof(f));
369 fclose(f);
370 mmap_unlock();
372 #endif
374 #endif
377 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
379 PageDesc *pd;
380 void **lp;
381 int i;
383 #if defined(CONFIG_USER_ONLY)
384 /* We can't use g_malloc because it may recurse into a locked mutex. */
385 # define ALLOC(P, SIZE) \
386 do { \
387 P = mmap(NULL, SIZE, PROT_READ | PROT_WRITE, \
388 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); \
389 } while (0)
390 #else
391 # define ALLOC(P, SIZE) \
392 do { P = g_malloc0(SIZE); } while (0)
393 #endif
395 /* Level 1. Always allocated. */
396 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
398 /* Level 2..N-1. */
399 for (i = V_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
400 void **p = *lp;
402 if (p == NULL) {
403 if (!alloc) {
404 return NULL;
406 ALLOC(p, sizeof(void *) * L2_SIZE);
407 *lp = p;
410 lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
413 pd = *lp;
414 if (pd == NULL) {
415 if (!alloc) {
416 return NULL;
418 ALLOC(pd, sizeof(PageDesc) * L2_SIZE);
419 *lp = pd;
422 #undef ALLOC
424 return pd + (index & (L2_SIZE - 1));
427 static inline PageDesc *page_find(tb_page_addr_t index)
429 return page_find_alloc(index, 0);
432 #if !defined(CONFIG_USER_ONLY)
433 #define mmap_lock() do { } while (0)
434 #define mmap_unlock() do { } while (0)
435 #endif
437 #if defined(CONFIG_USER_ONLY)
438 /* Currently it is not recommended to allocate big chunks of data in
439 user mode. It will change when a dedicated libc will be used. */
440 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
441 region in which the guest needs to run. Revisit this. */
442 #define USE_STATIC_CODE_GEN_BUFFER
443 #endif
445 /* ??? Should configure for this, not list operating systems here. */
446 #if (defined(__linux__) \
447 || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
448 || defined(__DragonFly__) || defined(__OpenBSD__) \
449 || defined(__NetBSD__))
450 # define USE_MMAP
451 #endif
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__arm__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
466 #elif defined(__s390x__)
467 /* We have a +- 4GB range on the branches; leave some slop. */
468 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
469 #else
470 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
471 #endif
473 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
475 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
476 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
477 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
479 static inline size_t size_code_gen_buffer(size_t tb_size)
481 /* Size the buffer. */
482 if (tb_size == 0) {
483 #ifdef USE_STATIC_CODE_GEN_BUFFER
484 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
485 #else
486 /* ??? Needs adjustments. */
487 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
488 static buffer, we could size this on RESERVED_VA, on the text
489 segment size of the executable, or continue to use the default. */
490 tb_size = (unsigned long)(ram_size / 4);
491 #endif
493 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
494 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
496 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
497 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
499 tcg_ctx.code_gen_buffer_size = tb_size;
500 return tb_size;
503 #ifdef USE_STATIC_CODE_GEN_BUFFER
504 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
505 __attribute__((aligned(CODE_GEN_ALIGN)));
507 static inline void *alloc_code_gen_buffer(void)
509 map_exec(static_code_gen_buffer, tcg_ctx.code_gen_buffer_size);
510 return static_code_gen_buffer;
512 #elif defined(USE_MMAP)
513 static inline void *alloc_code_gen_buffer(void)
515 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
516 uintptr_t start = 0;
517 void *buf;
519 /* Constrain the position of the buffer based on the host cpu.
520 Note that these addresses are chosen in concert with the
521 addresses assigned in the relevant linker script file. */
522 # if defined(__PIE__) || defined(__PIC__)
523 /* Don't bother setting a preferred location if we're building
524 a position-independent executable. We're more likely to get
525 an address near the main executable if we let the kernel
526 choose the address. */
527 # elif defined(__x86_64__) && defined(MAP_32BIT)
528 /* Force the memory down into low memory with the executable.
529 Leave the choice of exact location with the kernel. */
530 flags |= MAP_32BIT;
531 /* Cannot expect to map more than 800MB in low memory. */
532 if (tcg_ctx.code_gen_buffer_size > 800u * 1024 * 1024) {
533 tcg_ctx.code_gen_buffer_size = 800u * 1024 * 1024;
535 # elif defined(__sparc__)
536 start = 0x40000000ul;
537 # elif defined(__s390x__)
538 start = 0x90000000ul;
539 # endif
541 buf = mmap((void *)start, tcg_ctx.code_gen_buffer_size,
542 PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
543 return buf == MAP_FAILED ? NULL : buf;
545 #else
546 static inline void *alloc_code_gen_buffer(void)
548 void *buf = g_malloc(tcg_ctx.code_gen_buffer_size);
550 if (buf) {
551 map_exec(buf, tcg_ctx.code_gen_buffer_size);
553 return buf;
555 #endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
557 static inline void code_gen_alloc(size_t tb_size)
559 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
560 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
561 if (tcg_ctx.code_gen_buffer == NULL) {
562 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
563 exit(1);
566 qemu_madvise(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size,
567 QEMU_MADV_HUGEPAGE);
569 /* Steal room for the prologue at the end of the buffer. This ensures
570 (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
571 from TB's to the prologue are going to be in range. It also means
572 that we don't need to mark (additional) portions of the data segment
573 as executable. */
574 tcg_ctx.code_gen_prologue = tcg_ctx.code_gen_buffer +
575 tcg_ctx.code_gen_buffer_size - 1024;
576 tcg_ctx.code_gen_buffer_size -= 1024;
578 tcg_ctx.code_gen_buffer_max_size = tcg_ctx.code_gen_buffer_size -
579 (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
580 tcg_ctx.code_gen_max_blocks = tcg_ctx.code_gen_buffer_size /
581 CODE_GEN_AVG_BLOCK_SIZE;
582 tcg_ctx.tb_ctx.tbs =
583 g_malloc(tcg_ctx.code_gen_max_blocks * sizeof(TranslationBlock));
586 /* Must be called before using the QEMU cpus. 'tb_size' is the size
587 (in bytes) allocated to the translation buffer. Zero means default
588 size. */
589 void tcg_exec_init(unsigned long tb_size)
591 cpu_gen_init();
592 code_gen_alloc(tb_size);
593 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
594 tcg_register_jit(tcg_ctx.code_gen_buffer, tcg_ctx.code_gen_buffer_size);
595 page_init();
596 #if !defined(CONFIG_USER_ONLY) || !defined(CONFIG_USE_GUEST_BASE)
597 /* There's no guest base to take into account, so go ahead and
598 initialize the prologue now. */
599 tcg_prologue_init(&tcg_ctx);
600 #endif
603 bool tcg_enabled(void)
605 return tcg_ctx.code_gen_buffer != NULL;
608 /* Allocate a new translation block. Flush the translation buffer if
609 too many translation blocks or too much generated code. */
610 static TranslationBlock *tb_alloc(target_ulong pc)
612 TranslationBlock *tb;
614 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks ||
615 (tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer) >=
616 tcg_ctx.code_gen_buffer_max_size) {
617 return NULL;
619 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
620 tb->pc = pc;
621 tb->cflags = 0;
622 return tb;
625 void tb_free(TranslationBlock *tb)
627 /* In practice this is mostly used for single use temporary TB
628 Ignore the hard cases and just back up if this TB happens to
629 be the last one generated. */
630 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
631 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
632 tcg_ctx.code_gen_ptr = tb->tc_ptr;
633 tcg_ctx.tb_ctx.nb_tbs--;
637 static inline void invalidate_page_bitmap(PageDesc *p)
639 if (p->code_bitmap) {
640 g_free(p->code_bitmap);
641 p->code_bitmap = NULL;
643 p->code_write_count = 0;
646 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
647 static void page_flush_tb_1(int level, void **lp)
649 int i;
651 if (*lp == NULL) {
652 return;
654 if (level == 0) {
655 PageDesc *pd = *lp;
657 for (i = 0; i < L2_SIZE; ++i) {
658 pd[i].first_tb = NULL;
659 invalidate_page_bitmap(pd + i);
661 } else {
662 void **pp = *lp;
664 for (i = 0; i < L2_SIZE; ++i) {
665 page_flush_tb_1(level - 1, pp + i);
670 static void page_flush_tb(void)
672 int i;
674 for (i = 0; i < V_L1_SIZE; i++) {
675 page_flush_tb_1(V_L1_SHIFT / L2_BITS - 1, l1_map + i);
679 /* flush all the translation blocks */
680 /* XXX: tb_flush is currently not thread safe */
681 void tb_flush(CPUArchState *env1)
683 CPUArchState *env;
685 #if defined(DEBUG_FLUSH)
686 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
687 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
688 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
689 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
690 tcg_ctx.tb_ctx.nb_tbs : 0);
691 #endif
692 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
693 > tcg_ctx.code_gen_buffer_size) {
694 cpu_abort(env1, "Internal error: code buffer overflow\n");
696 tcg_ctx.tb_ctx.nb_tbs = 0;
698 for (env = first_cpu; env != NULL; env = env->next_cpu) {
699 memset(env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof(void *));
702 memset(tcg_ctx.tb_ctx.tb_phys_hash, 0,
703 CODE_GEN_PHYS_HASH_SIZE * sizeof(void *));
704 page_flush_tb();
706 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
707 /* XXX: flush processor icache at this point if cache flush is
708 expensive */
709 tcg_ctx.tb_ctx.tb_flush_count++;
712 #ifdef DEBUG_TB_CHECK
714 static void tb_invalidate_check(target_ulong address)
716 TranslationBlock *tb;
717 int i;
719 address &= TARGET_PAGE_MASK;
720 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
721 for (tb = tb_ctx.tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
722 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
723 address >= tb->pc + tb->size)) {
724 printf("ERROR invalidate: address=" TARGET_FMT_lx
725 " PC=%08lx size=%04x\n",
726 address, (long)tb->pc, tb->size);
732 /* verify that all the pages have correct rights for code */
733 static void tb_page_check(void)
735 TranslationBlock *tb;
736 int i, flags1, flags2;
738 for (i = 0; i < CODE_GEN_PHYS_HASH_SIZE; i++) {
739 for (tb = tcg_ctx.tb_ctx.tb_phys_hash[i]; tb != NULL;
740 tb = tb->phys_hash_next) {
741 flags1 = page_get_flags(tb->pc);
742 flags2 = page_get_flags(tb->pc + tb->size - 1);
743 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
744 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
745 (long)tb->pc, tb->size, flags1, flags2);
751 #endif
753 static inline void tb_hash_remove(TranslationBlock **ptb, TranslationBlock *tb)
755 TranslationBlock *tb1;
757 for (;;) {
758 tb1 = *ptb;
759 if (tb1 == tb) {
760 *ptb = tb1->phys_hash_next;
761 break;
763 ptb = &tb1->phys_hash_next;
767 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
769 TranslationBlock *tb1;
770 unsigned int n1;
772 for (;;) {
773 tb1 = *ptb;
774 n1 = (uintptr_t)tb1 & 3;
775 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
776 if (tb1 == tb) {
777 *ptb = tb1->page_next[n1];
778 break;
780 ptb = &tb1->page_next[n1];
784 static inline void tb_jmp_remove(TranslationBlock *tb, int n)
786 TranslationBlock *tb1, **ptb;
787 unsigned int n1;
789 ptb = &tb->jmp_next[n];
790 tb1 = *ptb;
791 if (tb1) {
792 /* find tb(n) in circular list */
793 for (;;) {
794 tb1 = *ptb;
795 n1 = (uintptr_t)tb1 & 3;
796 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
797 if (n1 == n && tb1 == tb) {
798 break;
800 if (n1 == 2) {
801 ptb = &tb1->jmp_first;
802 } else {
803 ptb = &tb1->jmp_next[n1];
806 /* now we can suppress tb(n) from the list */
807 *ptb = tb->jmp_next[n];
809 tb->jmp_next[n] = NULL;
813 /* reset the jump entry 'n' of a TB so that it is not chained to
814 another TB */
815 static inline void tb_reset_jump(TranslationBlock *tb, int n)
817 tb_set_jmp_target(tb, n, (uintptr_t)(tb->tc_ptr + tb->tb_next_offset[n]));
820 /* invalidate one TB */
821 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
823 CPUArchState *env;
824 PageDesc *p;
825 unsigned int h, n1;
826 tb_page_addr_t phys_pc;
827 TranslationBlock *tb1, *tb2;
829 /* remove the TB from the hash list */
830 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
831 h = tb_phys_hash_func(phys_pc);
832 tb_hash_remove(&tcg_ctx.tb_ctx.tb_phys_hash[h], tb);
834 /* remove the TB from the page list */
835 if (tb->page_addr[0] != page_addr) {
836 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
837 tb_page_remove(&p->first_tb, tb);
838 invalidate_page_bitmap(p);
840 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
841 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
842 tb_page_remove(&p->first_tb, tb);
843 invalidate_page_bitmap(p);
846 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
848 /* remove the TB from the hash list */
849 h = tb_jmp_cache_hash_func(tb->pc);
850 for (env = first_cpu; env != NULL; env = env->next_cpu) {
851 if (env->tb_jmp_cache[h] == tb) {
852 env->tb_jmp_cache[h] = NULL;
856 /* suppress this TB from the two jump lists */
857 tb_jmp_remove(tb, 0);
858 tb_jmp_remove(tb, 1);
860 /* suppress any remaining jumps to this TB */
861 tb1 = tb->jmp_first;
862 for (;;) {
863 n1 = (uintptr_t)tb1 & 3;
864 if (n1 == 2) {
865 break;
867 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
868 tb2 = tb1->jmp_next[n1];
869 tb_reset_jump(tb1, n1);
870 tb1->jmp_next[n1] = NULL;
871 tb1 = tb2;
873 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2); /* fail safe */
875 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
878 static inline void set_bits(uint8_t *tab, int start, int len)
880 int end, mask, end1;
882 end = start + len;
883 tab += start >> 3;
884 mask = 0xff << (start & 7);
885 if ((start & ~7) == (end & ~7)) {
886 if (start < end) {
887 mask &= ~(0xff << (end & 7));
888 *tab |= mask;
890 } else {
891 *tab++ |= mask;
892 start = (start + 8) & ~7;
893 end1 = end & ~7;
894 while (start < end1) {
895 *tab++ = 0xff;
896 start += 8;
898 if (start < end) {
899 mask = ~(0xff << (end & 7));
900 *tab |= mask;
905 static void build_page_bitmap(PageDesc *p)
907 int n, tb_start, tb_end;
908 TranslationBlock *tb;
910 p->code_bitmap = g_malloc0(TARGET_PAGE_SIZE / 8);
912 tb = p->first_tb;
913 while (tb != NULL) {
914 n = (uintptr_t)tb & 3;
915 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
916 /* NOTE: this is subtle as a TB may span two physical pages */
917 if (n == 0) {
918 /* NOTE: tb_end may be after the end of the page, but
919 it is not a problem */
920 tb_start = tb->pc & ~TARGET_PAGE_MASK;
921 tb_end = tb_start + tb->size;
922 if (tb_end > TARGET_PAGE_SIZE) {
923 tb_end = TARGET_PAGE_SIZE;
925 } else {
926 tb_start = 0;
927 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
929 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
930 tb = tb->page_next[n];
934 TranslationBlock *tb_gen_code(CPUArchState *env,
935 target_ulong pc, target_ulong cs_base,
936 int flags, int cflags)
938 TranslationBlock *tb;
939 uint8_t *tc_ptr;
940 tb_page_addr_t phys_pc, phys_page2;
941 target_ulong virt_page2;
942 int code_gen_size;
944 phys_pc = get_page_addr_code(env, pc);
945 tb = tb_alloc(pc);
946 if (!tb) {
947 /* flush must be done */
948 tb_flush(env);
949 /* cannot fail at this point */
950 tb = tb_alloc(pc);
951 /* Don't forget to invalidate previous TB info. */
952 tcg_ctx.tb_ctx.tb_invalidated_flag = 1;
954 tc_ptr = tcg_ctx.code_gen_ptr;
955 tb->tc_ptr = tc_ptr;
956 tb->cs_base = cs_base;
957 tb->flags = flags;
958 tb->cflags = cflags;
959 cpu_gen_code(env, tb, &code_gen_size);
960 tcg_ctx.code_gen_ptr = (void *)(((uintptr_t)tcg_ctx.code_gen_ptr +
961 code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
963 /* check next page if needed */
964 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
965 phys_page2 = -1;
966 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
967 phys_page2 = get_page_addr_code(env, virt_page2);
969 tb_link_page(tb, phys_pc, phys_page2);
970 return tb;
974 * Invalidate all TBs which intersect with the target physical address range
975 * [start;end[. NOTE: start and end may refer to *different* physical pages.
976 * 'is_cpu_write_access' should be true if called from a real cpu write
977 * access: the virtual CPU will exit the current TB if code is modified inside
978 * this TB.
980 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
981 int is_cpu_write_access)
983 while (start < end) {
984 tb_invalidate_phys_page_range(start, end, is_cpu_write_access);
985 start &= TARGET_PAGE_MASK;
986 start += TARGET_PAGE_SIZE;
991 * Invalidate all TBs which intersect with the target physical address range
992 * [start;end[. NOTE: start and end must refer to the *same* physical page.
993 * 'is_cpu_write_access' should be true if called from a real cpu write
994 * access: the virtual CPU will exit the current TB if code is modified inside
995 * this TB.
997 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
998 int is_cpu_write_access)
1000 TranslationBlock *tb, *tb_next, *saved_tb;
1001 CPUArchState *env = cpu_single_env;
1002 CPUState *cpu = NULL;
1003 tb_page_addr_t tb_start, tb_end;
1004 PageDesc *p;
1005 int n;
1006 #ifdef TARGET_HAS_PRECISE_SMC
1007 int current_tb_not_found = is_cpu_write_access;
1008 TranslationBlock *current_tb = NULL;
1009 int current_tb_modified = 0;
1010 target_ulong current_pc = 0;
1011 target_ulong current_cs_base = 0;
1012 int current_flags = 0;
1013 #endif /* TARGET_HAS_PRECISE_SMC */
1015 p = page_find(start >> TARGET_PAGE_BITS);
1016 if (!p) {
1017 return;
1019 if (!p->code_bitmap &&
1020 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
1021 is_cpu_write_access) {
1022 /* build code bitmap */
1023 build_page_bitmap(p);
1025 if (env != NULL) {
1026 cpu = ENV_GET_CPU(env);
1029 /* we remove all the TBs in the range [start, end[ */
1030 /* XXX: see if in some cases it could be faster to invalidate all
1031 the code */
1032 tb = p->first_tb;
1033 while (tb != NULL) {
1034 n = (uintptr_t)tb & 3;
1035 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1036 tb_next = tb->page_next[n];
1037 /* NOTE: this is subtle as a TB may span two physical pages */
1038 if (n == 0) {
1039 /* NOTE: tb_end may be after the end of the page, but
1040 it is not a problem */
1041 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1042 tb_end = tb_start + tb->size;
1043 } else {
1044 tb_start = tb->page_addr[1];
1045 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1047 if (!(tb_end <= start || tb_start >= end)) {
1048 #ifdef TARGET_HAS_PRECISE_SMC
1049 if (current_tb_not_found) {
1050 current_tb_not_found = 0;
1051 current_tb = NULL;
1052 if (env->mem_io_pc) {
1053 /* now we have a real cpu fault */
1054 current_tb = tb_find_pc(env->mem_io_pc);
1057 if (current_tb == tb &&
1058 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1059 /* If we are modifying the current TB, we must stop
1060 its execution. We could be more precise by checking
1061 that the modification is after the current PC, but it
1062 would require a specialized function to partially
1063 restore the CPU state */
1065 current_tb_modified = 1;
1066 cpu_restore_state_from_tb(current_tb, env, env->mem_io_pc);
1067 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1068 &current_flags);
1070 #endif /* TARGET_HAS_PRECISE_SMC */
1071 /* we need to do that to handle the case where a signal
1072 occurs while doing tb_phys_invalidate() */
1073 saved_tb = NULL;
1074 if (cpu != NULL) {
1075 saved_tb = cpu->current_tb;
1076 cpu->current_tb = NULL;
1078 tb_phys_invalidate(tb, -1);
1079 if (cpu != NULL) {
1080 cpu->current_tb = saved_tb;
1081 if (cpu->interrupt_request && cpu->current_tb) {
1082 cpu_interrupt(cpu, cpu->interrupt_request);
1086 tb = tb_next;
1088 #if !defined(CONFIG_USER_ONLY)
1089 /* if no code remaining, no need to continue to use slow writes */
1090 if (!p->first_tb) {
1091 invalidate_page_bitmap(p);
1092 if (is_cpu_write_access) {
1093 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
1096 #endif
1097 #ifdef TARGET_HAS_PRECISE_SMC
1098 if (current_tb_modified) {
1099 /* we generate a block containing just the instruction
1100 modifying the memory. It will ensure that it cannot modify
1101 itself */
1102 cpu->current_tb = NULL;
1103 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1104 cpu_resume_from_signal(env, NULL);
1106 #endif
1109 /* len must be <= 8 and start must be a multiple of len */
1110 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1112 PageDesc *p;
1113 int offset, b;
1115 #if 0
1116 if (1) {
1117 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1118 cpu_single_env->mem_io_vaddr, len,
1119 cpu_single_env->eip,
1120 cpu_single_env->eip +
1121 (intptr_t)cpu_single_env->segs[R_CS].base);
1123 #endif
1124 p = page_find(start >> TARGET_PAGE_BITS);
1125 if (!p) {
1126 return;
1128 if (p->code_bitmap) {
1129 offset = start & ~TARGET_PAGE_MASK;
1130 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1131 if (b & ((1 << len) - 1)) {
1132 goto do_invalidate;
1134 } else {
1135 do_invalidate:
1136 tb_invalidate_phys_page_range(start, start + len, 1);
1140 #if !defined(CONFIG_SOFTMMU)
1141 static void tb_invalidate_phys_page(tb_page_addr_t addr,
1142 uintptr_t pc, void *puc)
1144 TranslationBlock *tb;
1145 PageDesc *p;
1146 int n;
1147 #ifdef TARGET_HAS_PRECISE_SMC
1148 TranslationBlock *current_tb = NULL;
1149 CPUArchState *env = cpu_single_env;
1150 CPUState *cpu = NULL;
1151 int current_tb_modified = 0;
1152 target_ulong current_pc = 0;
1153 target_ulong current_cs_base = 0;
1154 int current_flags = 0;
1155 #endif
1157 addr &= TARGET_PAGE_MASK;
1158 p = page_find(addr >> TARGET_PAGE_BITS);
1159 if (!p) {
1160 return;
1162 tb = p->first_tb;
1163 #ifdef TARGET_HAS_PRECISE_SMC
1164 if (tb && pc != 0) {
1165 current_tb = tb_find_pc(pc);
1167 if (env != NULL) {
1168 cpu = ENV_GET_CPU(env);
1170 #endif
1171 while (tb != NULL) {
1172 n = (uintptr_t)tb & 3;
1173 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1174 #ifdef TARGET_HAS_PRECISE_SMC
1175 if (current_tb == tb &&
1176 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1177 /* If we are modifying the current TB, we must stop
1178 its execution. We could be more precise by checking
1179 that the modification is after the current PC, but it
1180 would require a specialized function to partially
1181 restore the CPU state */
1183 current_tb_modified = 1;
1184 cpu_restore_state_from_tb(current_tb, env, pc);
1185 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1186 &current_flags);
1188 #endif /* TARGET_HAS_PRECISE_SMC */
1189 tb_phys_invalidate(tb, addr);
1190 tb = tb->page_next[n];
1192 p->first_tb = NULL;
1193 #ifdef TARGET_HAS_PRECISE_SMC
1194 if (current_tb_modified) {
1195 /* we generate a block containing just the instruction
1196 modifying the memory. It will ensure that it cannot modify
1197 itself */
1198 cpu->current_tb = NULL;
1199 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
1200 cpu_resume_from_signal(env, puc);
1202 #endif
1204 #endif
1206 /* add the tb in the target page and protect it if necessary */
1207 static inline void tb_alloc_page(TranslationBlock *tb,
1208 unsigned int n, tb_page_addr_t page_addr)
1210 PageDesc *p;
1211 #ifndef CONFIG_USER_ONLY
1212 bool page_already_protected;
1213 #endif
1215 tb->page_addr[n] = page_addr;
1216 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1217 tb->page_next[n] = p->first_tb;
1218 #ifndef CONFIG_USER_ONLY
1219 page_already_protected = p->first_tb != NULL;
1220 #endif
1221 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1222 invalidate_page_bitmap(p);
1224 #if defined(TARGET_HAS_SMC) || 1
1226 #if defined(CONFIG_USER_ONLY)
1227 if (p->flags & PAGE_WRITE) {
1228 target_ulong addr;
1229 PageDesc *p2;
1230 int prot;
1232 /* force the host page as non writable (writes will have a
1233 page fault + mprotect overhead) */
1234 page_addr &= qemu_host_page_mask;
1235 prot = 0;
1236 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1237 addr += TARGET_PAGE_SIZE) {
1239 p2 = page_find(addr >> TARGET_PAGE_BITS);
1240 if (!p2) {
1241 continue;
1243 prot |= p2->flags;
1244 p2->flags &= ~PAGE_WRITE;
1246 mprotect(g2h(page_addr), qemu_host_page_size,
1247 (prot & PAGE_BITS) & ~PAGE_WRITE);
1248 #ifdef DEBUG_TB_INVALIDATE
1249 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1250 page_addr);
1251 #endif
1253 #else
1254 /* if some code is already present, then the pages are already
1255 protected. So we handle the case where only the first TB is
1256 allocated in a physical page */
1257 if (!page_already_protected) {
1258 tlb_protect_code(page_addr);
1260 #endif
1262 #endif /* TARGET_HAS_SMC */
1265 /* add a new TB and link it to the physical page tables. phys_page2 is
1266 (-1) to indicate that only one page contains the TB. */
1267 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1268 tb_page_addr_t phys_page2)
1270 unsigned int h;
1271 TranslationBlock **ptb;
1273 /* Grab the mmap lock to stop another thread invalidating this TB
1274 before we are done. */
1275 mmap_lock();
1276 /* add in the physical hash table */
1277 h = tb_phys_hash_func(phys_pc);
1278 ptb = &tcg_ctx.tb_ctx.tb_phys_hash[h];
1279 tb->phys_hash_next = *ptb;
1280 *ptb = tb;
1282 /* add in the page list */
1283 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1284 if (phys_page2 != -1) {
1285 tb_alloc_page(tb, 1, phys_page2);
1286 } else {
1287 tb->page_addr[1] = -1;
1290 tb->jmp_first = (TranslationBlock *)((uintptr_t)tb | 2);
1291 tb->jmp_next[0] = NULL;
1292 tb->jmp_next[1] = NULL;
1294 /* init original jump addresses */
1295 if (tb->tb_next_offset[0] != 0xffff) {
1296 tb_reset_jump(tb, 0);
1298 if (tb->tb_next_offset[1] != 0xffff) {
1299 tb_reset_jump(tb, 1);
1302 #ifdef DEBUG_TB_CHECK
1303 tb_page_check();
1304 #endif
1305 mmap_unlock();
1308 #if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
1309 /* check whether the given addr is in TCG generated code buffer or not */
1310 bool is_tcg_gen_code(uintptr_t tc_ptr)
1312 /* This can be called during code generation, code_gen_buffer_size
1313 is used instead of code_gen_ptr for upper boundary checking */
1314 return (tc_ptr >= (uintptr_t)tcg_ctx.code_gen_buffer &&
1315 tc_ptr < (uintptr_t)(tcg_ctx.code_gen_buffer +
1316 tcg_ctx.code_gen_buffer_size));
1318 #endif
1320 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1321 tb[1].tc_ptr. Return NULL if not found */
1322 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1324 int m_min, m_max, m;
1325 uintptr_t v;
1326 TranslationBlock *tb;
1328 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1329 return NULL;
1331 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1332 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1333 return NULL;
1335 /* binary search (cf Knuth) */
1336 m_min = 0;
1337 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1338 while (m_min <= m_max) {
1339 m = (m_min + m_max) >> 1;
1340 tb = &tcg_ctx.tb_ctx.tbs[m];
1341 v = (uintptr_t)tb->tc_ptr;
1342 if (v == tc_ptr) {
1343 return tb;
1344 } else if (tc_ptr < v) {
1345 m_max = m - 1;
1346 } else {
1347 m_min = m + 1;
1350 return &tcg_ctx.tb_ctx.tbs[m_max];
1353 #if defined(TARGET_HAS_ICE) && !defined(CONFIG_USER_ONLY)
1354 void tb_invalidate_phys_addr(hwaddr addr)
1356 ram_addr_t ram_addr;
1357 MemoryRegionSection *section;
1359 section = phys_page_find(address_space_memory.dispatch,
1360 addr >> TARGET_PAGE_BITS);
1361 if (!(memory_region_is_ram(section->mr)
1362 || (section->mr->rom_device && section->mr->readable))) {
1363 return;
1365 ram_addr = (memory_region_get_ram_addr(section->mr) & TARGET_PAGE_MASK)
1366 + memory_region_section_addr(section, addr);
1367 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1369 #endif /* TARGET_HAS_ICE && !defined(CONFIG_USER_ONLY) */
1371 void tb_check_watchpoint(CPUArchState *env)
1373 TranslationBlock *tb;
1375 tb = tb_find_pc(env->mem_io_pc);
1376 if (!tb) {
1377 cpu_abort(env, "check_watchpoint: could not find TB for pc=%p",
1378 (void *)env->mem_io_pc);
1380 cpu_restore_state_from_tb(tb, env, env->mem_io_pc);
1381 tb_phys_invalidate(tb, -1);
1384 #ifndef CONFIG_USER_ONLY
1385 /* mask must never be zero, except for A20 change call */
1386 static void tcg_handle_interrupt(CPUState *cpu, int mask)
1388 CPUArchState *env = cpu->env_ptr;
1389 int old_mask;
1391 old_mask = cpu->interrupt_request;
1392 cpu->interrupt_request |= mask;
1395 * If called from iothread context, wake the target cpu in
1396 * case its halted.
1398 if (!qemu_cpu_is_self(cpu)) {
1399 qemu_cpu_kick(cpu);
1400 return;
1403 if (use_icount) {
1404 env->icount_decr.u16.high = 0xffff;
1405 if (!can_do_io(env)
1406 && (mask & ~old_mask) != 0) {
1407 cpu_abort(env, "Raised interrupt while not in I/O function");
1409 } else {
1410 cpu->tcg_exit_req = 1;
1414 CPUInterruptHandler cpu_interrupt_handler = tcg_handle_interrupt;
1416 /* in deterministic execution mode, instructions doing device I/Os
1417 must be at the end of the TB */
1418 void cpu_io_recompile(CPUArchState *env, uintptr_t retaddr)
1420 TranslationBlock *tb;
1421 uint32_t n, cflags;
1422 target_ulong pc, cs_base;
1423 uint64_t flags;
1425 tb = tb_find_pc(retaddr);
1426 if (!tb) {
1427 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
1428 (void *)retaddr);
1430 n = env->icount_decr.u16.low + tb->icount;
1431 cpu_restore_state_from_tb(tb, env, retaddr);
1432 /* Calculate how many instructions had been executed before the fault
1433 occurred. */
1434 n = n - env->icount_decr.u16.low;
1435 /* Generate a new TB ending on the I/O insn. */
1436 n++;
1437 /* On MIPS and SH, delay slot instructions can only be restarted if
1438 they were already the first instruction in the TB. If this is not
1439 the first instruction in a TB then re-execute the preceding
1440 branch. */
1441 #if defined(TARGET_MIPS)
1442 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1443 env->active_tc.PC -= 4;
1444 env->icount_decr.u16.low++;
1445 env->hflags &= ~MIPS_HFLAG_BMASK;
1447 #elif defined(TARGET_SH4)
1448 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1449 && n > 1) {
1450 env->pc -= 2;
1451 env->icount_decr.u16.low++;
1452 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1454 #endif
1455 /* This should never happen. */
1456 if (n > CF_COUNT_MASK) {
1457 cpu_abort(env, "TB too big during recompile");
1460 cflags = n | CF_LAST_IO;
1461 pc = tb->pc;
1462 cs_base = tb->cs_base;
1463 flags = tb->flags;
1464 tb_phys_invalidate(tb, -1);
1465 /* FIXME: In theory this could raise an exception. In practice
1466 we have already translated the block once so it's probably ok. */
1467 tb_gen_code(env, pc, cs_base, flags, cflags);
1468 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1469 the first in the TB) then we end up generating a whole new TB and
1470 repeating the fault, which is horribly inefficient.
1471 Better would be to execute just this insn uncached, or generate a
1472 second new TB. */
1473 cpu_resume_from_signal(env, NULL);
1476 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr)
1478 unsigned int i;
1480 /* Discard jump cache entries for any tb which might potentially
1481 overlap the flushed page. */
1482 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1483 memset(&env->tb_jmp_cache[i], 0,
1484 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1486 i = tb_jmp_cache_hash_page(addr);
1487 memset(&env->tb_jmp_cache[i], 0,
1488 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1491 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1493 int i, target_code_size, max_target_code_size;
1494 int direct_jmp_count, direct_jmp2_count, cross_page;
1495 TranslationBlock *tb;
1497 target_code_size = 0;
1498 max_target_code_size = 0;
1499 cross_page = 0;
1500 direct_jmp_count = 0;
1501 direct_jmp2_count = 0;
1502 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1503 tb = &tcg_ctx.tb_ctx.tbs[i];
1504 target_code_size += tb->size;
1505 if (tb->size > max_target_code_size) {
1506 max_target_code_size = tb->size;
1508 if (tb->page_addr[1] != -1) {
1509 cross_page++;
1511 if (tb->tb_next_offset[0] != 0xffff) {
1512 direct_jmp_count++;
1513 if (tb->tb_next_offset[1] != 0xffff) {
1514 direct_jmp2_count++;
1518 /* XXX: avoid using doubles ? */
1519 cpu_fprintf(f, "Translation buffer state:\n");
1520 cpu_fprintf(f, "gen code size %td/%zd\n",
1521 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1522 tcg_ctx.code_gen_buffer_max_size);
1523 cpu_fprintf(f, "TB count %d/%d\n",
1524 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1525 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1526 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1527 tcg_ctx.tb_ctx.nb_tbs : 0,
1528 max_target_code_size);
1529 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1530 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1531 tcg_ctx.code_gen_buffer) /
1532 tcg_ctx.tb_ctx.nb_tbs : 0,
1533 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1534 tcg_ctx.code_gen_buffer) /
1535 target_code_size : 0);
1536 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1537 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1538 tcg_ctx.tb_ctx.nb_tbs : 0);
1539 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1540 direct_jmp_count,
1541 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1542 tcg_ctx.tb_ctx.nb_tbs : 0,
1543 direct_jmp2_count,
1544 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1545 tcg_ctx.tb_ctx.nb_tbs : 0);
1546 cpu_fprintf(f, "\nStatistics:\n");
1547 cpu_fprintf(f, "TB flush count %d\n", tcg_ctx.tb_ctx.tb_flush_count);
1548 cpu_fprintf(f, "TB invalidate count %d\n",
1549 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1550 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1551 tcg_dump_info(f, cpu_fprintf);
1554 #else /* CONFIG_USER_ONLY */
1556 void cpu_interrupt(CPUState *cpu, int mask)
1558 cpu->interrupt_request |= mask;
1559 cpu->tcg_exit_req = 1;
1563 * Walks guest process memory "regions" one by one
1564 * and calls callback function 'fn' for each region.
1566 struct walk_memory_regions_data {
1567 walk_memory_regions_fn fn;
1568 void *priv;
1569 uintptr_t start;
1570 int prot;
1573 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1574 abi_ulong end, int new_prot)
1576 if (data->start != -1ul) {
1577 int rc = data->fn(data->priv, data->start, end, data->prot);
1578 if (rc != 0) {
1579 return rc;
1583 data->start = (new_prot ? end : -1ul);
1584 data->prot = new_prot;
1586 return 0;
1589 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1590 abi_ulong base, int level, void **lp)
1592 abi_ulong pa;
1593 int i, rc;
1595 if (*lp == NULL) {
1596 return walk_memory_regions_end(data, base, 0);
1599 if (level == 0) {
1600 PageDesc *pd = *lp;
1602 for (i = 0; i < L2_SIZE; ++i) {
1603 int prot = pd[i].flags;
1605 pa = base | (i << TARGET_PAGE_BITS);
1606 if (prot != data->prot) {
1607 rc = walk_memory_regions_end(data, pa, prot);
1608 if (rc != 0) {
1609 return rc;
1613 } else {
1614 void **pp = *lp;
1616 for (i = 0; i < L2_SIZE; ++i) {
1617 pa = base | ((abi_ulong)i <<
1618 (TARGET_PAGE_BITS + L2_BITS * level));
1619 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1620 if (rc != 0) {
1621 return rc;
1626 return 0;
1629 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1631 struct walk_memory_regions_data data;
1632 uintptr_t i;
1634 data.fn = fn;
1635 data.priv = priv;
1636 data.start = -1ul;
1637 data.prot = 0;
1639 for (i = 0; i < V_L1_SIZE; i++) {
1640 int rc = walk_memory_regions_1(&data, (abi_ulong)i << V_L1_SHIFT,
1641 V_L1_SHIFT / L2_BITS - 1, l1_map + i);
1643 if (rc != 0) {
1644 return rc;
1648 return walk_memory_regions_end(&data, 0, 0);
1651 static int dump_region(void *priv, abi_ulong start,
1652 abi_ulong end, unsigned long prot)
1654 FILE *f = (FILE *)priv;
1656 (void) fprintf(f, TARGET_ABI_FMT_lx"-"TARGET_ABI_FMT_lx
1657 " "TARGET_ABI_FMT_lx" %c%c%c\n",
1658 start, end, end - start,
1659 ((prot & PAGE_READ) ? 'r' : '-'),
1660 ((prot & PAGE_WRITE) ? 'w' : '-'),
1661 ((prot & PAGE_EXEC) ? 'x' : '-'));
1663 return 0;
1666 /* dump memory mappings */
1667 void page_dump(FILE *f)
1669 (void) fprintf(f, "%-8s %-8s %-8s %s\n",
1670 "start", "end", "size", "prot");
1671 walk_memory_regions(f, dump_region);
1674 int page_get_flags(target_ulong address)
1676 PageDesc *p;
1678 p = page_find(address >> TARGET_PAGE_BITS);
1679 if (!p) {
1680 return 0;
1682 return p->flags;
1685 /* Modify the flags of a page and invalidate the code if necessary.
1686 The flag PAGE_WRITE_ORG is positioned automatically depending
1687 on PAGE_WRITE. The mmap_lock should already be held. */
1688 void page_set_flags(target_ulong start, target_ulong end, int flags)
1690 target_ulong addr, len;
1692 /* This function should never be called with addresses outside the
1693 guest address space. If this assert fires, it probably indicates
1694 a missing call to h2g_valid. */
1695 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1696 assert(end < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1697 #endif
1698 assert(start < end);
1700 start = start & TARGET_PAGE_MASK;
1701 end = TARGET_PAGE_ALIGN(end);
1703 if (flags & PAGE_WRITE) {
1704 flags |= PAGE_WRITE_ORG;
1707 for (addr = start, len = end - start;
1708 len != 0;
1709 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1710 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1712 /* If the write protection bit is set, then we invalidate
1713 the code inside. */
1714 if (!(p->flags & PAGE_WRITE) &&
1715 (flags & PAGE_WRITE) &&
1716 p->first_tb) {
1717 tb_invalidate_phys_page(addr, 0, NULL);
1719 p->flags = flags;
1723 int page_check_range(target_ulong start, target_ulong len, int flags)
1725 PageDesc *p;
1726 target_ulong end;
1727 target_ulong addr;
1729 /* This function should never be called with addresses outside the
1730 guest address space. If this assert fires, it probably indicates
1731 a missing call to h2g_valid. */
1732 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1733 assert(start < ((abi_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1734 #endif
1736 if (len == 0) {
1737 return 0;
1739 if (start + len - 1 < start) {
1740 /* We've wrapped around. */
1741 return -1;
1744 /* must do before we loose bits in the next step */
1745 end = TARGET_PAGE_ALIGN(start + len);
1746 start = start & TARGET_PAGE_MASK;
1748 for (addr = start, len = end - start;
1749 len != 0;
1750 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1751 p = page_find(addr >> TARGET_PAGE_BITS);
1752 if (!p) {
1753 return -1;
1755 if (!(p->flags & PAGE_VALID)) {
1756 return -1;
1759 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
1760 return -1;
1762 if (flags & PAGE_WRITE) {
1763 if (!(p->flags & PAGE_WRITE_ORG)) {
1764 return -1;
1766 /* unprotect the page if it was put read-only because it
1767 contains translated code */
1768 if (!(p->flags & PAGE_WRITE)) {
1769 if (!page_unprotect(addr, 0, NULL)) {
1770 return -1;
1773 return 0;
1776 return 0;
1779 /* called from signal handler: invalidate the code and unprotect the
1780 page. Return TRUE if the fault was successfully handled. */
1781 int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
1783 unsigned int prot;
1784 PageDesc *p;
1785 target_ulong host_start, host_end, addr;
1787 /* Technically this isn't safe inside a signal handler. However we
1788 know this only ever happens in a synchronous SEGV handler, so in
1789 practice it seems to be ok. */
1790 mmap_lock();
1792 p = page_find(address >> TARGET_PAGE_BITS);
1793 if (!p) {
1794 mmap_unlock();
1795 return 0;
1798 /* if the page was really writable, then we change its
1799 protection back to writable */
1800 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
1801 host_start = address & qemu_host_page_mask;
1802 host_end = host_start + qemu_host_page_size;
1804 prot = 0;
1805 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
1806 p = page_find(addr >> TARGET_PAGE_BITS);
1807 p->flags |= PAGE_WRITE;
1808 prot |= p->flags;
1810 /* and since the content will be modified, we must invalidate
1811 the corresponding translated code. */
1812 tb_invalidate_phys_page(addr, pc, puc);
1813 #ifdef DEBUG_TB_CHECK
1814 tb_invalidate_check(addr);
1815 #endif
1817 mprotect((void *)g2h(host_start), qemu_host_page_size,
1818 prot & PAGE_BITS);
1820 mmap_unlock();
1821 return 1;
1823 mmap_unlock();
1824 return 0;
1826 #endif /* CONFIG_USER_ONLY */