crypto: extend mode as a parameter in qcrypto_cipher_supports()
[qemu/ar7.git] / translate-all.c
blob8ca393c9d0ab6f2bd7cbf62ccfbae48402e3be11
1 /*
2 * Host code generation
4 * Copyright (c) 2003 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19 #ifdef _WIN32
20 #include <windows.h>
21 #endif
22 #include "qemu/osdep.h"
25 #include "qemu-common.h"
26 #define NO_CPU_IO_DEFS
27 #include "cpu.h"
28 #include "trace.h"
29 #include "disas/disas.h"
30 #include "exec/exec-all.h"
31 #include "tcg.h"
32 #if defined(CONFIG_USER_ONLY)
33 #include "qemu.h"
34 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
35 #include <sys/param.h>
36 #if __FreeBSD_version >= 700104
37 #define HAVE_KINFO_GETVMMAP
38 #define sigqueue sigqueue_freebsd /* avoid redefinition */
39 #include <sys/proc.h>
40 #include <machine/profile.h>
41 #define _KERNEL
42 #include <sys/user.h>
43 #undef _KERNEL
44 #undef sigqueue
45 #include <libutil.h>
46 #endif
47 #endif
48 #else
49 #include "exec/address-spaces.h"
50 #endif
52 #include "exec/cputlb.h"
53 #include "exec/tb-hash.h"
54 #include "translate-all.h"
55 #include "qemu/bitmap.h"
56 #include "qemu/timer.h"
57 #include "exec/log.h"
59 //#define DEBUG_TB_INVALIDATE
60 //#define DEBUG_FLUSH
61 /* make various TB consistency checks */
62 //#define DEBUG_TB_CHECK
64 #if !defined(CONFIG_USER_ONLY)
65 /* TB consistency checks only implemented for usermode emulation. */
66 #undef DEBUG_TB_CHECK
67 #endif
69 #define SMC_BITMAP_USE_THRESHOLD 10
71 typedef struct PageDesc {
72 /* list of TBs intersecting this ram page */
73 TranslationBlock *first_tb;
74 #ifdef CONFIG_SOFTMMU
75 /* in order to optimize self modifying code, we count the number
76 of lookups we do to a given page to use a bitmap */
77 unsigned int code_write_count;
78 unsigned long *code_bitmap;
79 #else
80 unsigned long flags;
81 #endif
82 } PageDesc;
84 /* In system mode we want L1_MAP to be based on ram offsets,
85 while in user mode we want it to be based on virtual addresses. */
86 #if !defined(CONFIG_USER_ONLY)
87 #if HOST_LONG_BITS < TARGET_PHYS_ADDR_SPACE_BITS
88 # define L1_MAP_ADDR_SPACE_BITS HOST_LONG_BITS
89 #else
90 # define L1_MAP_ADDR_SPACE_BITS TARGET_PHYS_ADDR_SPACE_BITS
91 #endif
92 #else
93 # define L1_MAP_ADDR_SPACE_BITS TARGET_VIRT_ADDR_SPACE_BITS
94 #endif
96 /* Size of the L2 (and L3, etc) page tables. */
97 #define V_L2_BITS 10
98 #define V_L2_SIZE (1 << V_L2_BITS)
100 /* The bits remaining after N lower levels of page tables. */
101 #define V_L1_BITS_REM \
102 ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % V_L2_BITS)
104 #if V_L1_BITS_REM < 4
105 #define V_L1_BITS (V_L1_BITS_REM + V_L2_BITS)
106 #else
107 #define V_L1_BITS V_L1_BITS_REM
108 #endif
110 #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
112 #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
114 uintptr_t qemu_host_page_size;
115 intptr_t qemu_host_page_mask;
117 /* The bottom level has pointers to PageDesc */
118 static void *l1_map[V_L1_SIZE];
120 /* code generation context */
121 TCGContext tcg_ctx;
123 /* translation block context */
124 #ifdef CONFIG_USER_ONLY
125 __thread int have_tb_lock;
126 #endif
128 void tb_lock(void)
130 #ifdef CONFIG_USER_ONLY
131 assert(!have_tb_lock);
132 qemu_mutex_lock(&tcg_ctx.tb_ctx.tb_lock);
133 have_tb_lock++;
134 #endif
137 void tb_unlock(void)
139 #ifdef CONFIG_USER_ONLY
140 assert(have_tb_lock);
141 have_tb_lock--;
142 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
143 #endif
146 void tb_lock_reset(void)
148 #ifdef CONFIG_USER_ONLY
149 if (have_tb_lock) {
150 qemu_mutex_unlock(&tcg_ctx.tb_ctx.tb_lock);
151 have_tb_lock = 0;
153 #endif
156 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr);
158 void cpu_gen_init(void)
160 tcg_context_init(&tcg_ctx);
163 /* Encode VAL as a signed leb128 sequence at P.
164 Return P incremented past the encoded value. */
165 static uint8_t *encode_sleb128(uint8_t *p, target_long val)
167 int more, byte;
169 do {
170 byte = val & 0x7f;
171 val >>= 7;
172 more = !((val == 0 && (byte & 0x40) == 0)
173 || (val == -1 && (byte & 0x40) != 0));
174 if (more) {
175 byte |= 0x80;
177 *p++ = byte;
178 } while (more);
180 return p;
183 /* Decode a signed leb128 sequence at *PP; increment *PP past the
184 decoded value. Return the decoded value. */
185 static target_long decode_sleb128(uint8_t **pp)
187 uint8_t *p = *pp;
188 target_long val = 0;
189 int byte, shift = 0;
191 do {
192 byte = *p++;
193 val |= (target_ulong)(byte & 0x7f) << shift;
194 shift += 7;
195 } while (byte & 0x80);
196 if (shift < TARGET_LONG_BITS && (byte & 0x40)) {
197 val |= -(target_ulong)1 << shift;
200 *pp = p;
201 return val;
204 /* Encode the data collected about the instructions while compiling TB.
205 Place the data at BLOCK, and return the number of bytes consumed.
207 The logical table consisits of TARGET_INSN_START_WORDS target_ulong's,
208 which come from the target's insn_start data, followed by a uintptr_t
209 which comes from the host pc of the end of the code implementing the insn.
211 Each line of the table is encoded as sleb128 deltas from the previous
212 line. The seed for the first line is { tb->pc, 0..., tb->tc_ptr }.
213 That is, the first column is seeded with the guest pc, the last column
214 with the host pc, and the middle columns with zeros. */
216 static int encode_search(TranslationBlock *tb, uint8_t *block)
218 uint8_t *highwater = tcg_ctx.code_gen_highwater;
219 uint8_t *p = block;
220 int i, j, n;
222 tb->tc_search = block;
224 for (i = 0, n = tb->icount; i < n; ++i) {
225 target_ulong prev;
227 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
228 if (i == 0) {
229 prev = (j == 0 ? tb->pc : 0);
230 } else {
231 prev = tcg_ctx.gen_insn_data[i - 1][j];
233 p = encode_sleb128(p, tcg_ctx.gen_insn_data[i][j] - prev);
235 prev = (i == 0 ? 0 : tcg_ctx.gen_insn_end_off[i - 1]);
236 p = encode_sleb128(p, tcg_ctx.gen_insn_end_off[i] - prev);
238 /* Test for (pending) buffer overflow. The assumption is that any
239 one row beginning below the high water mark cannot overrun
240 the buffer completely. Thus we can test for overflow after
241 encoding a row without having to check during encoding. */
242 if (unlikely(p > highwater)) {
243 return -1;
247 return p - block;
250 /* The cpu state corresponding to 'searched_pc' is restored. */
251 static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
252 uintptr_t searched_pc)
254 target_ulong data[TARGET_INSN_START_WORDS] = { tb->pc };
255 uintptr_t host_pc = (uintptr_t)tb->tc_ptr;
256 CPUArchState *env = cpu->env_ptr;
257 uint8_t *p = tb->tc_search;
258 int i, j, num_insns = tb->icount;
259 #ifdef CONFIG_PROFILER
260 int64_t ti = profile_getclock();
261 #endif
263 searched_pc -= GETPC_ADJ;
265 if (searched_pc < host_pc) {
266 return -1;
269 /* Reconstruct the stored insn data while looking for the point at
270 which the end of the insn exceeds the searched_pc. */
271 for (i = 0; i < num_insns; ++i) {
272 for (j = 0; j < TARGET_INSN_START_WORDS; ++j) {
273 data[j] += decode_sleb128(&p);
275 host_pc += decode_sleb128(&p);
276 if (host_pc > searched_pc) {
277 goto found;
280 return -1;
282 found:
283 if (tb->cflags & CF_USE_ICOUNT) {
284 assert(use_icount);
285 /* Reset the cycle counter to the start of the block. */
286 cpu->icount_decr.u16.low += num_insns;
287 /* Clear the IO flag. */
288 cpu->can_do_io = 0;
290 cpu->icount_decr.u16.low -= i;
291 restore_state_to_opc(env, tb, data);
293 #ifdef CONFIG_PROFILER
294 tcg_ctx.restore_time += profile_getclock() - ti;
295 tcg_ctx.restore_count++;
296 #endif
297 return 0;
300 bool cpu_restore_state(CPUState *cpu, uintptr_t retaddr)
302 TranslationBlock *tb;
304 tb = tb_find_pc(retaddr);
305 if (tb) {
306 cpu_restore_state_from_tb(cpu, tb, retaddr);
307 if (tb->cflags & CF_NOCACHE) {
308 /* one-shot translation, invalidate it immediately */
309 tb_phys_invalidate(tb, -1);
310 tb_free(tb);
312 return true;
314 return false;
317 void page_size_init(void)
319 /* NOTE: we can always suppose that qemu_host_page_size >=
320 TARGET_PAGE_SIZE */
321 qemu_real_host_page_size = getpagesize();
322 qemu_real_host_page_mask = -(intptr_t)qemu_real_host_page_size;
323 if (qemu_host_page_size == 0) {
324 qemu_host_page_size = qemu_real_host_page_size;
326 if (qemu_host_page_size < TARGET_PAGE_SIZE) {
327 qemu_host_page_size = TARGET_PAGE_SIZE;
329 qemu_host_page_mask = -(intptr_t)qemu_host_page_size;
332 static void page_init(void)
334 page_size_init();
335 #if defined(CONFIG_BSD) && defined(CONFIG_USER_ONLY)
337 #ifdef HAVE_KINFO_GETVMMAP
338 struct kinfo_vmentry *freep;
339 int i, cnt;
341 freep = kinfo_getvmmap(getpid(), &cnt);
342 if (freep) {
343 mmap_lock();
344 for (i = 0; i < cnt; i++) {
345 unsigned long startaddr, endaddr;
347 startaddr = freep[i].kve_start;
348 endaddr = freep[i].kve_end;
349 if (h2g_valid(startaddr)) {
350 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
352 if (h2g_valid(endaddr)) {
353 endaddr = h2g(endaddr);
354 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
355 } else {
356 #if TARGET_ABI_BITS <= L1_MAP_ADDR_SPACE_BITS
357 endaddr = ~0ul;
358 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
359 #endif
363 free(freep);
364 mmap_unlock();
366 #else
367 FILE *f;
369 last_brk = (unsigned long)sbrk(0);
371 f = fopen("/compat/linux/proc/self/maps", "r");
372 if (f) {
373 mmap_lock();
375 do {
376 unsigned long startaddr, endaddr;
377 int n;
379 n = fscanf(f, "%lx-%lx %*[^\n]\n", &startaddr, &endaddr);
381 if (n == 2 && h2g_valid(startaddr)) {
382 startaddr = h2g(startaddr) & TARGET_PAGE_MASK;
384 if (h2g_valid(endaddr)) {
385 endaddr = h2g(endaddr);
386 } else {
387 endaddr = ~0ul;
389 page_set_flags(startaddr, endaddr, PAGE_RESERVED);
391 } while (!feof(f));
393 fclose(f);
394 mmap_unlock();
396 #endif
398 #endif
401 /* If alloc=1:
402 * Called with mmap_lock held for user-mode emulation.
404 static PageDesc *page_find_alloc(tb_page_addr_t index, int alloc)
406 PageDesc *pd;
407 void **lp;
408 int i;
410 /* Level 1. Always allocated. */
411 lp = l1_map + ((index >> V_L1_SHIFT) & (V_L1_SIZE - 1));
413 /* Level 2..N-1. */
414 for (i = V_L1_SHIFT / V_L2_BITS - 1; i > 0; i--) {
415 void **p = atomic_rcu_read(lp);
417 if (p == NULL) {
418 if (!alloc) {
419 return NULL;
421 p = g_new0(void *, V_L2_SIZE);
422 atomic_rcu_set(lp, p);
425 lp = p + ((index >> (i * V_L2_BITS)) & (V_L2_SIZE - 1));
428 pd = atomic_rcu_read(lp);
429 if (pd == NULL) {
430 if (!alloc) {
431 return NULL;
433 pd = g_new0(PageDesc, V_L2_SIZE);
434 atomic_rcu_set(lp, pd);
437 return pd + (index & (V_L2_SIZE - 1));
440 static inline PageDesc *page_find(tb_page_addr_t index)
442 return page_find_alloc(index, 0);
445 #if defined(CONFIG_USER_ONLY)
446 /* Currently it is not recommended to allocate big chunks of data in
447 user mode. It will change when a dedicated libc will be used. */
448 /* ??? 64-bit hosts ought to have no problem mmaping data outside the
449 region in which the guest needs to run. Revisit this. */
450 #define USE_STATIC_CODE_GEN_BUFFER
451 #endif
453 /* Minimum size of the code gen buffer. This number is randomly chosen,
454 but not so small that we can't have a fair number of TB's live. */
455 #define MIN_CODE_GEN_BUFFER_SIZE (1024u * 1024)
457 /* Maximum size of the code gen buffer we'd like to use. Unless otherwise
458 indicated, this is constrained by the range of direct branches on the
459 host cpu, as used by the TCG implementation of goto_tb. */
460 #if defined(__x86_64__)
461 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
462 #elif defined(__sparc__)
463 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
464 #elif defined(__powerpc64__)
465 # define MAX_CODE_GEN_BUFFER_SIZE (2ul * 1024 * 1024 * 1024)
466 #elif defined(__powerpc__)
467 # define MAX_CODE_GEN_BUFFER_SIZE (32u * 1024 * 1024)
468 #elif defined(__aarch64__)
469 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
470 #elif defined(__arm__)
471 # define MAX_CODE_GEN_BUFFER_SIZE (16u * 1024 * 1024)
472 #elif defined(__s390x__)
473 /* We have a +- 4GB range on the branches; leave some slop. */
474 # define MAX_CODE_GEN_BUFFER_SIZE (3ul * 1024 * 1024 * 1024)
475 #elif defined(__mips__)
476 /* We have a 256MB branch region, but leave room to make sure the
477 main executable is also within that region. */
478 # define MAX_CODE_GEN_BUFFER_SIZE (128ul * 1024 * 1024)
479 #else
480 # define MAX_CODE_GEN_BUFFER_SIZE ((size_t)-1)
481 #endif
483 #define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
485 #define DEFAULT_CODE_GEN_BUFFER_SIZE \
486 (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
487 ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
489 static inline size_t size_code_gen_buffer(size_t tb_size)
491 /* Size the buffer. */
492 if (tb_size == 0) {
493 #ifdef USE_STATIC_CODE_GEN_BUFFER
494 tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
495 #else
496 /* ??? Needs adjustments. */
497 /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
498 static buffer, we could size this on RESERVED_VA, on the text
499 segment size of the executable, or continue to use the default. */
500 tb_size = (unsigned long)(ram_size / 4);
501 #endif
503 if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
504 tb_size = MIN_CODE_GEN_BUFFER_SIZE;
506 if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
507 tb_size = MAX_CODE_GEN_BUFFER_SIZE;
509 return tb_size;
512 #ifdef __mips__
513 /* In order to use J and JAL within the code_gen_buffer, we require
514 that the buffer not cross a 256MB boundary. */
515 static inline bool cross_256mb(void *addr, size_t size)
517 return ((uintptr_t)addr ^ ((uintptr_t)addr + size)) & ~0x0ffffffful;
520 /* We weren't able to allocate a buffer without crossing that boundary,
521 so make do with the larger portion of the buffer that doesn't cross.
522 Returns the new base of the buffer, and adjusts code_gen_buffer_size. */
523 static inline void *split_cross_256mb(void *buf1, size_t size1)
525 void *buf2 = (void *)(((uintptr_t)buf1 + size1) & ~0x0ffffffful);
526 size_t size2 = buf1 + size1 - buf2;
528 size1 = buf2 - buf1;
529 if (size1 < size2) {
530 size1 = size2;
531 buf1 = buf2;
534 tcg_ctx.code_gen_buffer_size = size1;
535 return buf1;
537 #endif
539 #ifdef USE_STATIC_CODE_GEN_BUFFER
540 static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
541 __attribute__((aligned(CODE_GEN_ALIGN)));
543 # ifdef _WIN32
544 static inline void do_protect(void *addr, long size, int prot)
546 DWORD old_protect;
547 VirtualProtect(addr, size, prot, &old_protect);
550 static inline void map_exec(void *addr, long size)
552 do_protect(addr, size, PAGE_EXECUTE_READWRITE);
555 static inline void map_none(void *addr, long size)
557 do_protect(addr, size, PAGE_NOACCESS);
559 # else
560 static inline void do_protect(void *addr, long size, int prot)
562 uintptr_t start, end;
564 start = (uintptr_t)addr;
565 start &= qemu_real_host_page_mask;
567 end = (uintptr_t)addr + size;
568 end = ROUND_UP(end, qemu_real_host_page_size);
570 mprotect((void *)start, end - start, prot);
573 static inline void map_exec(void *addr, long size)
575 do_protect(addr, size, PROT_READ | PROT_WRITE | PROT_EXEC);
578 static inline void map_none(void *addr, long size)
580 do_protect(addr, size, PROT_NONE);
582 # endif /* WIN32 */
584 static inline void *alloc_code_gen_buffer(void)
586 void *buf = static_code_gen_buffer;
587 size_t full_size, size;
589 /* The size of the buffer, rounded down to end on a page boundary. */
590 full_size = (((uintptr_t)buf + sizeof(static_code_gen_buffer))
591 & qemu_real_host_page_mask) - (uintptr_t)buf;
593 /* Reserve a guard page. */
594 size = full_size - qemu_real_host_page_size;
596 /* Honor a command-line option limiting the size of the buffer. */
597 if (size > tcg_ctx.code_gen_buffer_size) {
598 size = (((uintptr_t)buf + tcg_ctx.code_gen_buffer_size)
599 & qemu_real_host_page_mask) - (uintptr_t)buf;
601 tcg_ctx.code_gen_buffer_size = size;
603 #ifdef __mips__
604 if (cross_256mb(buf, size)) {
605 buf = split_cross_256mb(buf, size);
606 size = tcg_ctx.code_gen_buffer_size;
608 #endif
610 map_exec(buf, size);
611 map_none(buf + size, qemu_real_host_page_size);
612 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
614 return buf;
616 #elif defined(_WIN32)
617 static inline void *alloc_code_gen_buffer(void)
619 size_t size = tcg_ctx.code_gen_buffer_size;
620 void *buf1, *buf2;
622 /* Perform the allocation in two steps, so that the guard page
623 is reserved but uncommitted. */
624 buf1 = VirtualAlloc(NULL, size + qemu_real_host_page_size,
625 MEM_RESERVE, PAGE_NOACCESS);
626 if (buf1 != NULL) {
627 buf2 = VirtualAlloc(buf1, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
628 assert(buf1 == buf2);
631 return buf1;
633 #else
634 static inline void *alloc_code_gen_buffer(void)
636 int flags = MAP_PRIVATE | MAP_ANONYMOUS;
637 uintptr_t start = 0;
638 size_t size = tcg_ctx.code_gen_buffer_size;
639 void *buf;
641 /* Constrain the position of the buffer based on the host cpu.
642 Note that these addresses are chosen in concert with the
643 addresses assigned in the relevant linker script file. */
644 # if defined(__PIE__) || defined(__PIC__)
645 /* Don't bother setting a preferred location if we're building
646 a position-independent executable. We're more likely to get
647 an address near the main executable if we let the kernel
648 choose the address. */
649 # elif defined(__x86_64__) && defined(MAP_32BIT)
650 /* Force the memory down into low memory with the executable.
651 Leave the choice of exact location with the kernel. */
652 flags |= MAP_32BIT;
653 /* Cannot expect to map more than 800MB in low memory. */
654 if (size > 800u * 1024 * 1024) {
655 tcg_ctx.code_gen_buffer_size = size = 800u * 1024 * 1024;
657 # elif defined(__sparc__)
658 start = 0x40000000ul;
659 # elif defined(__s390x__)
660 start = 0x90000000ul;
661 # elif defined(__mips__)
662 # if _MIPS_SIM == _ABI64
663 start = 0x128000000ul;
664 # else
665 start = 0x08000000ul;
666 # endif
667 # endif
669 buf = mmap((void *)start, size + qemu_real_host_page_size,
670 PROT_NONE, flags, -1, 0);
671 if (buf == MAP_FAILED) {
672 return NULL;
675 #ifdef __mips__
676 if (cross_256mb(buf, size)) {
677 /* Try again, with the original still mapped, to avoid re-acquiring
678 that 256mb crossing. This time don't specify an address. */
679 size_t size2;
680 void *buf2 = mmap(NULL, size + qemu_real_host_page_size,
681 PROT_NONE, flags, -1, 0);
682 switch (buf2 != MAP_FAILED) {
683 case 1:
684 if (!cross_256mb(buf2, size)) {
685 /* Success! Use the new buffer. */
686 munmap(buf, size + qemu_real_host_page_size);
687 break;
689 /* Failure. Work with what we had. */
690 munmap(buf2, size + qemu_real_host_page_size);
691 /* fallthru */
692 default:
693 /* Split the original buffer. Free the smaller half. */
694 buf2 = split_cross_256mb(buf, size);
695 size2 = tcg_ctx.code_gen_buffer_size;
696 if (buf == buf2) {
697 munmap(buf + size2 + qemu_real_host_page_size, size - size2);
698 } else {
699 munmap(buf, size - size2);
701 size = size2;
702 break;
704 buf = buf2;
706 #endif
708 /* Make the final buffer accessible. The guard page at the end
709 will remain inaccessible with PROT_NONE. */
710 mprotect(buf, size, PROT_WRITE | PROT_READ | PROT_EXEC);
712 /* Request large pages for the buffer. */
713 qemu_madvise(buf, size, QEMU_MADV_HUGEPAGE);
715 return buf;
717 #endif /* USE_STATIC_CODE_GEN_BUFFER, WIN32, POSIX */
719 static inline void code_gen_alloc(size_t tb_size)
721 tcg_ctx.code_gen_buffer_size = size_code_gen_buffer(tb_size);
722 tcg_ctx.code_gen_buffer = alloc_code_gen_buffer();
723 if (tcg_ctx.code_gen_buffer == NULL) {
724 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
725 exit(1);
728 /* Estimate a good size for the number of TBs we can support. We
729 still haven't deducted the prologue from the buffer size here,
730 but that's minimal and won't affect the estimate much. */
731 tcg_ctx.code_gen_max_blocks
732 = tcg_ctx.code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
733 tcg_ctx.tb_ctx.tbs = g_new(TranslationBlock, tcg_ctx.code_gen_max_blocks);
735 qemu_mutex_init(&tcg_ctx.tb_ctx.tb_lock);
738 static void tb_htable_init(void)
740 unsigned int mode = QHT_MODE_AUTO_RESIZE;
742 qht_init(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE, mode);
745 /* Must be called before using the QEMU cpus. 'tb_size' is the size
746 (in bytes) allocated to the translation buffer. Zero means default
747 size. */
748 void tcg_exec_init(unsigned long tb_size)
750 cpu_gen_init();
751 page_init();
752 tb_htable_init();
753 code_gen_alloc(tb_size);
754 #if defined(CONFIG_SOFTMMU)
755 /* There's no guest base to take into account, so go ahead and
756 initialize the prologue now. */
757 tcg_prologue_init(&tcg_ctx);
758 #endif
761 bool tcg_enabled(void)
763 return tcg_ctx.code_gen_buffer != NULL;
766 /* Allocate a new translation block. Flush the translation buffer if
767 too many translation blocks or too much generated code. */
768 static TranslationBlock *tb_alloc(target_ulong pc)
770 TranslationBlock *tb;
772 if (tcg_ctx.tb_ctx.nb_tbs >= tcg_ctx.code_gen_max_blocks) {
773 return NULL;
775 tb = &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs++];
776 tb->pc = pc;
777 tb->cflags = 0;
778 tb->invalid = false;
779 return tb;
782 void tb_free(TranslationBlock *tb)
784 /* In practice this is mostly used for single use temporary TB
785 Ignore the hard cases and just back up if this TB happens to
786 be the last one generated. */
787 if (tcg_ctx.tb_ctx.nb_tbs > 0 &&
788 tb == &tcg_ctx.tb_ctx.tbs[tcg_ctx.tb_ctx.nb_tbs - 1]) {
789 tcg_ctx.code_gen_ptr = tb->tc_ptr;
790 tcg_ctx.tb_ctx.nb_tbs--;
794 static inline void invalidate_page_bitmap(PageDesc *p)
796 #ifdef CONFIG_SOFTMMU
797 g_free(p->code_bitmap);
798 p->code_bitmap = NULL;
799 p->code_write_count = 0;
800 #endif
803 /* Set to NULL all the 'first_tb' fields in all PageDescs. */
804 static void page_flush_tb_1(int level, void **lp)
806 int i;
808 if (*lp == NULL) {
809 return;
811 if (level == 0) {
812 PageDesc *pd = *lp;
814 for (i = 0; i < V_L2_SIZE; ++i) {
815 pd[i].first_tb = NULL;
816 invalidate_page_bitmap(pd + i);
818 } else {
819 void **pp = *lp;
821 for (i = 0; i < V_L2_SIZE; ++i) {
822 page_flush_tb_1(level - 1, pp + i);
827 static void page_flush_tb(void)
829 int i;
831 for (i = 0; i < V_L1_SIZE; i++) {
832 page_flush_tb_1(V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
836 /* flush all the translation blocks */
837 static void do_tb_flush(CPUState *cpu, void *data)
839 unsigned tb_flush_req = (unsigned) (uintptr_t) data;
841 tb_lock();
843 /* If it's already been done on request of another CPU,
844 * just retry.
846 if (tcg_ctx.tb_ctx.tb_flush_count != tb_flush_req) {
847 goto done;
850 #if defined(DEBUG_FLUSH)
851 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
852 (unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer),
853 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.tb_ctx.nb_tbs > 0 ?
854 ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)) /
855 tcg_ctx.tb_ctx.nb_tbs : 0);
856 #endif
857 if ((unsigned long)(tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer)
858 > tcg_ctx.code_gen_buffer_size) {
859 cpu_abort(cpu, "Internal error: code buffer overflow\n");
862 CPU_FOREACH(cpu) {
863 int i;
865 for (i = 0; i < TB_JMP_CACHE_SIZE; ++i) {
866 atomic_set(&cpu->tb_jmp_cache[i], NULL);
870 tcg_ctx.tb_ctx.nb_tbs = 0;
871 qht_reset_size(&tcg_ctx.tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
872 page_flush_tb();
874 tcg_ctx.code_gen_ptr = tcg_ctx.code_gen_buffer;
875 /* XXX: flush processor icache at this point if cache flush is
876 expensive */
877 atomic_mb_set(&tcg_ctx.tb_ctx.tb_flush_count,
878 tcg_ctx.tb_ctx.tb_flush_count + 1);
880 done:
881 tb_unlock();
884 void tb_flush(CPUState *cpu)
886 if (tcg_enabled()) {
887 uintptr_t tb_flush_req = atomic_mb_read(&tcg_ctx.tb_ctx.tb_flush_count);
888 async_safe_run_on_cpu(cpu, do_tb_flush, (void *) tb_flush_req);
892 #ifdef DEBUG_TB_CHECK
894 static void
895 do_tb_invalidate_check(struct qht *ht, void *p, uint32_t hash, void *userp)
897 TranslationBlock *tb = p;
898 target_ulong addr = *(target_ulong *)userp;
900 if (!(addr + TARGET_PAGE_SIZE <= tb->pc || addr >= tb->pc + tb->size)) {
901 printf("ERROR invalidate: address=" TARGET_FMT_lx
902 " PC=%08lx size=%04x\n", addr, (long)tb->pc, tb->size);
906 static void tb_invalidate_check(target_ulong address)
908 address &= TARGET_PAGE_MASK;
909 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_invalidate_check, &address);
912 static void
913 do_tb_page_check(struct qht *ht, void *p, uint32_t hash, void *userp)
915 TranslationBlock *tb = p;
916 int flags1, flags2;
918 flags1 = page_get_flags(tb->pc);
919 flags2 = page_get_flags(tb->pc + tb->size - 1);
920 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
921 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
922 (long)tb->pc, tb->size, flags1, flags2);
926 /* verify that all the pages have correct rights for code */
927 static void tb_page_check(void)
929 qht_iter(&tcg_ctx.tb_ctx.htable, do_tb_page_check, NULL);
932 #endif
934 static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
936 TranslationBlock *tb1;
937 unsigned int n1;
939 for (;;) {
940 tb1 = *ptb;
941 n1 = (uintptr_t)tb1 & 3;
942 tb1 = (TranslationBlock *)((uintptr_t)tb1 & ~3);
943 if (tb1 == tb) {
944 *ptb = tb1->page_next[n1];
945 break;
947 ptb = &tb1->page_next[n1];
951 /* remove the TB from a list of TBs jumping to the n-th jump target of the TB */
952 static inline void tb_remove_from_jmp_list(TranslationBlock *tb, int n)
954 TranslationBlock *tb1;
955 uintptr_t *ptb, ntb;
956 unsigned int n1;
958 ptb = &tb->jmp_list_next[n];
959 if (*ptb) {
960 /* find tb(n) in circular list */
961 for (;;) {
962 ntb = *ptb;
963 n1 = ntb & 3;
964 tb1 = (TranslationBlock *)(ntb & ~3);
965 if (n1 == n && tb1 == tb) {
966 break;
968 if (n1 == 2) {
969 ptb = &tb1->jmp_list_first;
970 } else {
971 ptb = &tb1->jmp_list_next[n1];
974 /* now we can suppress tb(n) from the list */
975 *ptb = tb->jmp_list_next[n];
977 tb->jmp_list_next[n] = (uintptr_t)NULL;
981 /* reset the jump entry 'n' of a TB so that it is not chained to
982 another TB */
983 static inline void tb_reset_jump(TranslationBlock *tb, int n)
985 uintptr_t addr = (uintptr_t)(tb->tc_ptr + tb->jmp_reset_offset[n]);
986 tb_set_jmp_target(tb, n, addr);
989 /* remove any jumps to the TB */
990 static inline void tb_jmp_unlink(TranslationBlock *tb)
992 TranslationBlock *tb1;
993 uintptr_t *ptb, ntb;
994 unsigned int n1;
996 ptb = &tb->jmp_list_first;
997 for (;;) {
998 ntb = *ptb;
999 n1 = ntb & 3;
1000 tb1 = (TranslationBlock *)(ntb & ~3);
1001 if (n1 == 2) {
1002 break;
1004 tb_reset_jump(tb1, n1);
1005 *ptb = tb1->jmp_list_next[n1];
1006 tb1->jmp_list_next[n1] = (uintptr_t)NULL;
1010 /* invalidate one TB */
1011 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
1013 CPUState *cpu;
1014 PageDesc *p;
1015 uint32_t h;
1016 tb_page_addr_t phys_pc;
1018 atomic_set(&tb->invalid, true);
1020 /* remove the TB from the hash list */
1021 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1022 h = tb_hash_func(phys_pc, tb->pc, tb->flags);
1023 qht_remove(&tcg_ctx.tb_ctx.htable, tb, h);
1025 /* remove the TB from the page list */
1026 if (tb->page_addr[0] != page_addr) {
1027 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
1028 tb_page_remove(&p->first_tb, tb);
1029 invalidate_page_bitmap(p);
1031 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
1032 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
1033 tb_page_remove(&p->first_tb, tb);
1034 invalidate_page_bitmap(p);
1037 /* remove the TB from the hash list */
1038 h = tb_jmp_cache_hash_func(tb->pc);
1039 CPU_FOREACH(cpu) {
1040 if (atomic_read(&cpu->tb_jmp_cache[h]) == tb) {
1041 atomic_set(&cpu->tb_jmp_cache[h], NULL);
1045 /* suppress this TB from the two jump lists */
1046 tb_remove_from_jmp_list(tb, 0);
1047 tb_remove_from_jmp_list(tb, 1);
1049 /* suppress any remaining jumps to this TB */
1050 tb_jmp_unlink(tb);
1052 tcg_ctx.tb_ctx.tb_phys_invalidate_count++;
1055 #ifdef CONFIG_SOFTMMU
1056 static void build_page_bitmap(PageDesc *p)
1058 int n, tb_start, tb_end;
1059 TranslationBlock *tb;
1061 p->code_bitmap = bitmap_new(TARGET_PAGE_SIZE);
1063 tb = p->first_tb;
1064 while (tb != NULL) {
1065 n = (uintptr_t)tb & 3;
1066 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1067 /* NOTE: this is subtle as a TB may span two physical pages */
1068 if (n == 0) {
1069 /* NOTE: tb_end may be after the end of the page, but
1070 it is not a problem */
1071 tb_start = tb->pc & ~TARGET_PAGE_MASK;
1072 tb_end = tb_start + tb->size;
1073 if (tb_end > TARGET_PAGE_SIZE) {
1074 tb_end = TARGET_PAGE_SIZE;
1076 } else {
1077 tb_start = 0;
1078 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1080 bitmap_set(p->code_bitmap, tb_start, tb_end - tb_start);
1081 tb = tb->page_next[n];
1084 #endif
1086 /* add the tb in the target page and protect it if necessary
1088 * Called with mmap_lock held for user-mode emulation.
1090 static inline void tb_alloc_page(TranslationBlock *tb,
1091 unsigned int n, tb_page_addr_t page_addr)
1093 PageDesc *p;
1094 #ifndef CONFIG_USER_ONLY
1095 bool page_already_protected;
1096 #endif
1098 tb->page_addr[n] = page_addr;
1099 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS, 1);
1100 tb->page_next[n] = p->first_tb;
1101 #ifndef CONFIG_USER_ONLY
1102 page_already_protected = p->first_tb != NULL;
1103 #endif
1104 p->first_tb = (TranslationBlock *)((uintptr_t)tb | n);
1105 invalidate_page_bitmap(p);
1107 #if defined(CONFIG_USER_ONLY)
1108 if (p->flags & PAGE_WRITE) {
1109 target_ulong addr;
1110 PageDesc *p2;
1111 int prot;
1113 /* force the host page as non writable (writes will have a
1114 page fault + mprotect overhead) */
1115 page_addr &= qemu_host_page_mask;
1116 prot = 0;
1117 for (addr = page_addr; addr < page_addr + qemu_host_page_size;
1118 addr += TARGET_PAGE_SIZE) {
1120 p2 = page_find(addr >> TARGET_PAGE_BITS);
1121 if (!p2) {
1122 continue;
1124 prot |= p2->flags;
1125 p2->flags &= ~PAGE_WRITE;
1127 mprotect(g2h(page_addr), qemu_host_page_size,
1128 (prot & PAGE_BITS) & ~PAGE_WRITE);
1129 #ifdef DEBUG_TB_INVALIDATE
1130 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
1131 page_addr);
1132 #endif
1134 #else
1135 /* if some code is already present, then the pages are already
1136 protected. So we handle the case where only the first TB is
1137 allocated in a physical page */
1138 if (!page_already_protected) {
1139 tlb_protect_code(page_addr);
1141 #endif
1144 /* add a new TB and link it to the physical page tables. phys_page2 is
1145 * (-1) to indicate that only one page contains the TB.
1147 * Called with mmap_lock held for user-mode emulation.
1149 static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
1150 tb_page_addr_t phys_page2)
1152 uint32_t h;
1154 /* add in the page list */
1155 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1156 if (phys_page2 != -1) {
1157 tb_alloc_page(tb, 1, phys_page2);
1158 } else {
1159 tb->page_addr[1] = -1;
1162 /* add in the hash table */
1163 h = tb_hash_func(phys_pc, tb->pc, tb->flags);
1164 qht_insert(&tcg_ctx.tb_ctx.htable, tb, h);
1166 #ifdef DEBUG_TB_CHECK
1167 tb_page_check();
1168 #endif
1171 /* Called with mmap_lock held for user mode emulation. */
1172 TranslationBlock *tb_gen_code(CPUState *cpu,
1173 target_ulong pc, target_ulong cs_base,
1174 uint32_t flags, int cflags)
1176 CPUArchState *env = cpu->env_ptr;
1177 TranslationBlock *tb;
1178 tb_page_addr_t phys_pc, phys_page2;
1179 target_ulong virt_page2;
1180 tcg_insn_unit *gen_code_buf;
1181 int gen_code_size, search_size;
1182 #ifdef CONFIG_PROFILER
1183 int64_t ti;
1184 #endif
1186 phys_pc = get_page_addr_code(env, pc);
1187 if (use_icount && !(cflags & CF_IGNORE_ICOUNT)) {
1188 cflags |= CF_USE_ICOUNT;
1191 tb = tb_alloc(pc);
1192 if (unlikely(!tb)) {
1193 buffer_overflow:
1194 /* flush must be done */
1195 tb_flush(cpu);
1196 mmap_unlock();
1197 cpu_loop_exit(cpu);
1200 gen_code_buf = tcg_ctx.code_gen_ptr;
1201 tb->tc_ptr = gen_code_buf;
1202 tb->cs_base = cs_base;
1203 tb->flags = flags;
1204 tb->cflags = cflags;
1206 #ifdef CONFIG_PROFILER
1207 tcg_ctx.tb_count1++; /* includes aborted translations because of
1208 exceptions */
1209 ti = profile_getclock();
1210 #endif
1212 tcg_func_start(&tcg_ctx);
1214 tcg_ctx.cpu = ENV_GET_CPU(env);
1215 gen_intermediate_code(env, tb);
1216 tcg_ctx.cpu = NULL;
1218 trace_translate_block(tb, tb->pc, tb->tc_ptr);
1220 /* generate machine code */
1221 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
1222 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
1223 tcg_ctx.tb_jmp_reset_offset = tb->jmp_reset_offset;
1224 #ifdef USE_DIRECT_JUMP
1225 tcg_ctx.tb_jmp_insn_offset = tb->jmp_insn_offset;
1226 tcg_ctx.tb_jmp_target_addr = NULL;
1227 #else
1228 tcg_ctx.tb_jmp_insn_offset = NULL;
1229 tcg_ctx.tb_jmp_target_addr = tb->jmp_target_addr;
1230 #endif
1232 #ifdef CONFIG_PROFILER
1233 tcg_ctx.tb_count++;
1234 tcg_ctx.interm_time += profile_getclock() - ti;
1235 tcg_ctx.code_time -= profile_getclock();
1236 #endif
1238 /* ??? Overflow could be handled better here. In particular, we
1239 don't need to re-do gen_intermediate_code, nor should we re-do
1240 the tcg optimization currently hidden inside tcg_gen_code. All
1241 that should be required is to flush the TBs, allocate a new TB,
1242 re-initialize it per above, and re-do the actual code generation. */
1243 gen_code_size = tcg_gen_code(&tcg_ctx, tb);
1244 if (unlikely(gen_code_size < 0)) {
1245 goto buffer_overflow;
1247 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size);
1248 if (unlikely(search_size < 0)) {
1249 goto buffer_overflow;
1252 #ifdef CONFIG_PROFILER
1253 tcg_ctx.code_time += profile_getclock();
1254 tcg_ctx.code_in_len += tb->size;
1255 tcg_ctx.code_out_len += gen_code_size;
1256 tcg_ctx.search_out_len += search_size;
1257 #endif
1259 #ifdef DEBUG_DISAS
1260 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) &&
1261 qemu_log_in_addr_range(tb->pc)) {
1262 qemu_log("OUT: [size=%d]\n", gen_code_size);
1263 log_disas(tb->tc_ptr, gen_code_size);
1264 qemu_log("\n");
1265 qemu_log_flush();
1267 #endif
1269 tcg_ctx.code_gen_ptr = (void *)
1270 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size,
1271 CODE_GEN_ALIGN);
1273 /* init jump list */
1274 assert(((uintptr_t)tb & 3) == 0);
1275 tb->jmp_list_first = (uintptr_t)tb | 2;
1276 tb->jmp_list_next[0] = (uintptr_t)NULL;
1277 tb->jmp_list_next[1] = (uintptr_t)NULL;
1279 /* init original jump addresses wich has been set during tcg_gen_code() */
1280 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1281 tb_reset_jump(tb, 0);
1283 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1284 tb_reset_jump(tb, 1);
1287 /* check next page if needed */
1288 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
1289 phys_page2 = -1;
1290 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
1291 phys_page2 = get_page_addr_code(env, virt_page2);
1293 /* As long as consistency of the TB stuff is provided by tb_lock in user
1294 * mode and is implicit in single-threaded softmmu emulation, no explicit
1295 * memory barrier is required before tb_link_page() makes the TB visible
1296 * through the physical hash table and physical page list.
1298 tb_link_page(tb, phys_pc, phys_page2);
1299 return tb;
1303 * Invalidate all TBs which intersect with the target physical address range
1304 * [start;end[. NOTE: start and end may refer to *different* physical pages.
1305 * 'is_cpu_write_access' should be true if called from a real cpu write
1306 * access: the virtual CPU will exit the current TB if code is modified inside
1307 * this TB.
1309 * Called with mmap_lock held for user-mode emulation
1311 void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
1313 while (start < end) {
1314 tb_invalidate_phys_page_range(start, end, 0);
1315 start &= TARGET_PAGE_MASK;
1316 start += TARGET_PAGE_SIZE;
1321 * Invalidate all TBs which intersect with the target physical address range
1322 * [start;end[. NOTE: start and end must refer to the *same* physical page.
1323 * 'is_cpu_write_access' should be true if called from a real cpu write
1324 * access: the virtual CPU will exit the current TB if code is modified inside
1325 * this TB.
1327 * Called with mmap_lock held for user-mode emulation
1329 void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end,
1330 int is_cpu_write_access)
1332 TranslationBlock *tb, *tb_next;
1333 #if defined(TARGET_HAS_PRECISE_SMC)
1334 CPUState *cpu = current_cpu;
1335 CPUArchState *env = NULL;
1336 #endif
1337 tb_page_addr_t tb_start, tb_end;
1338 PageDesc *p;
1339 int n;
1340 #ifdef TARGET_HAS_PRECISE_SMC
1341 int current_tb_not_found = is_cpu_write_access;
1342 TranslationBlock *current_tb = NULL;
1343 int current_tb_modified = 0;
1344 target_ulong current_pc = 0;
1345 target_ulong current_cs_base = 0;
1346 uint32_t current_flags = 0;
1347 #endif /* TARGET_HAS_PRECISE_SMC */
1349 p = page_find(start >> TARGET_PAGE_BITS);
1350 if (!p) {
1351 return;
1353 #if defined(TARGET_HAS_PRECISE_SMC)
1354 if (cpu != NULL) {
1355 env = cpu->env_ptr;
1357 #endif
1359 /* we remove all the TBs in the range [start, end[ */
1360 /* XXX: see if in some cases it could be faster to invalidate all
1361 the code */
1362 tb = p->first_tb;
1363 while (tb != NULL) {
1364 n = (uintptr_t)tb & 3;
1365 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1366 tb_next = tb->page_next[n];
1367 /* NOTE: this is subtle as a TB may span two physical pages */
1368 if (n == 0) {
1369 /* NOTE: tb_end may be after the end of the page, but
1370 it is not a problem */
1371 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
1372 tb_end = tb_start + tb->size;
1373 } else {
1374 tb_start = tb->page_addr[1];
1375 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
1377 if (!(tb_end <= start || tb_start >= end)) {
1378 #ifdef TARGET_HAS_PRECISE_SMC
1379 if (current_tb_not_found) {
1380 current_tb_not_found = 0;
1381 current_tb = NULL;
1382 if (cpu->mem_io_pc) {
1383 /* now we have a real cpu fault */
1384 current_tb = tb_find_pc(cpu->mem_io_pc);
1387 if (current_tb == tb &&
1388 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1389 /* If we are modifying the current TB, we must stop
1390 its execution. We could be more precise by checking
1391 that the modification is after the current PC, but it
1392 would require a specialized function to partially
1393 restore the CPU state */
1395 current_tb_modified = 1;
1396 cpu_restore_state_from_tb(cpu, current_tb, cpu->mem_io_pc);
1397 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1398 &current_flags);
1400 #endif /* TARGET_HAS_PRECISE_SMC */
1401 tb_phys_invalidate(tb, -1);
1403 tb = tb_next;
1405 #if !defined(CONFIG_USER_ONLY)
1406 /* if no code remaining, no need to continue to use slow writes */
1407 if (!p->first_tb) {
1408 invalidate_page_bitmap(p);
1409 tlb_unprotect_code(start);
1411 #endif
1412 #ifdef TARGET_HAS_PRECISE_SMC
1413 if (current_tb_modified) {
1414 /* we generate a block containing just the instruction
1415 modifying the memory. It will ensure that it cannot modify
1416 itself */
1417 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1418 cpu_loop_exit_noexc(cpu);
1420 #endif
1423 #ifdef CONFIG_SOFTMMU
1424 /* len must be <= 8 and start must be a multiple of len */
1425 void tb_invalidate_phys_page_fast(tb_page_addr_t start, int len)
1427 PageDesc *p;
1429 #if 0
1430 if (1) {
1431 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1432 cpu_single_env->mem_io_vaddr, len,
1433 cpu_single_env->eip,
1434 cpu_single_env->eip +
1435 (intptr_t)cpu_single_env->segs[R_CS].base);
1437 #endif
1438 p = page_find(start >> TARGET_PAGE_BITS);
1439 if (!p) {
1440 return;
1442 if (!p->code_bitmap &&
1443 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD) {
1444 /* build code bitmap */
1445 build_page_bitmap(p);
1447 if (p->code_bitmap) {
1448 unsigned int nr;
1449 unsigned long b;
1451 nr = start & ~TARGET_PAGE_MASK;
1452 b = p->code_bitmap[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG - 1));
1453 if (b & ((1 << len) - 1)) {
1454 goto do_invalidate;
1456 } else {
1457 do_invalidate:
1458 tb_invalidate_phys_page_range(start, start + len, 1);
1461 #else
1462 /* Called with mmap_lock held. If pc is not 0 then it indicates the
1463 * host PC of the faulting store instruction that caused this invalidate.
1464 * Returns true if the caller needs to abort execution of the current
1465 * TB (because it was modified by this store and the guest CPU has
1466 * precise-SMC semantics).
1468 static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
1470 TranslationBlock *tb;
1471 PageDesc *p;
1472 int n;
1473 #ifdef TARGET_HAS_PRECISE_SMC
1474 TranslationBlock *current_tb = NULL;
1475 CPUState *cpu = current_cpu;
1476 CPUArchState *env = NULL;
1477 int current_tb_modified = 0;
1478 target_ulong current_pc = 0;
1479 target_ulong current_cs_base = 0;
1480 uint32_t current_flags = 0;
1481 #endif
1483 addr &= TARGET_PAGE_MASK;
1484 p = page_find(addr >> TARGET_PAGE_BITS);
1485 if (!p) {
1486 return false;
1488 tb = p->first_tb;
1489 #ifdef TARGET_HAS_PRECISE_SMC
1490 if (tb && pc != 0) {
1491 current_tb = tb_find_pc(pc);
1493 if (cpu != NULL) {
1494 env = cpu->env_ptr;
1496 #endif
1497 while (tb != NULL) {
1498 n = (uintptr_t)tb & 3;
1499 tb = (TranslationBlock *)((uintptr_t)tb & ~3);
1500 #ifdef TARGET_HAS_PRECISE_SMC
1501 if (current_tb == tb &&
1502 (current_tb->cflags & CF_COUNT_MASK) != 1) {
1503 /* If we are modifying the current TB, we must stop
1504 its execution. We could be more precise by checking
1505 that the modification is after the current PC, but it
1506 would require a specialized function to partially
1507 restore the CPU state */
1509 current_tb_modified = 1;
1510 cpu_restore_state_from_tb(cpu, current_tb, pc);
1511 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1512 &current_flags);
1514 #endif /* TARGET_HAS_PRECISE_SMC */
1515 tb_phys_invalidate(tb, addr);
1516 tb = tb->page_next[n];
1518 p->first_tb = NULL;
1519 #ifdef TARGET_HAS_PRECISE_SMC
1520 if (current_tb_modified) {
1521 /* we generate a block containing just the instruction
1522 modifying the memory. It will ensure that it cannot modify
1523 itself */
1524 tb_gen_code(cpu, current_pc, current_cs_base, current_flags, 1);
1525 return true;
1527 #endif
1528 return false;
1530 #endif
1532 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1533 tb[1].tc_ptr. Return NULL if not found */
1534 static TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
1536 int m_min, m_max, m;
1537 uintptr_t v;
1538 TranslationBlock *tb;
1540 if (tcg_ctx.tb_ctx.nb_tbs <= 0) {
1541 return NULL;
1543 if (tc_ptr < (uintptr_t)tcg_ctx.code_gen_buffer ||
1544 tc_ptr >= (uintptr_t)tcg_ctx.code_gen_ptr) {
1545 return NULL;
1547 /* binary search (cf Knuth) */
1548 m_min = 0;
1549 m_max = tcg_ctx.tb_ctx.nb_tbs - 1;
1550 while (m_min <= m_max) {
1551 m = (m_min + m_max) >> 1;
1552 tb = &tcg_ctx.tb_ctx.tbs[m];
1553 v = (uintptr_t)tb->tc_ptr;
1554 if (v == tc_ptr) {
1555 return tb;
1556 } else if (tc_ptr < v) {
1557 m_max = m - 1;
1558 } else {
1559 m_min = m + 1;
1562 return &tcg_ctx.tb_ctx.tbs[m_max];
1565 #if !defined(CONFIG_USER_ONLY)
1566 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr)
1568 ram_addr_t ram_addr;
1569 MemoryRegion *mr;
1570 hwaddr l = 1;
1572 rcu_read_lock();
1573 mr = address_space_translate(as, addr, &addr, &l, false);
1574 if (!(memory_region_is_ram(mr)
1575 || memory_region_is_romd(mr))) {
1576 rcu_read_unlock();
1577 return;
1579 ram_addr = memory_region_get_ram_addr(mr) + addr;
1580 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
1581 rcu_read_unlock();
1583 #endif /* !defined(CONFIG_USER_ONLY) */
1585 void tb_check_watchpoint(CPUState *cpu)
1587 TranslationBlock *tb;
1589 tb = tb_find_pc(cpu->mem_io_pc);
1590 if (tb) {
1591 /* We can use retranslation to find the PC. */
1592 cpu_restore_state_from_tb(cpu, tb, cpu->mem_io_pc);
1593 tb_phys_invalidate(tb, -1);
1594 } else {
1595 /* The exception probably happened in a helper. The CPU state should
1596 have been saved before calling it. Fetch the PC from there. */
1597 CPUArchState *env = cpu->env_ptr;
1598 target_ulong pc, cs_base;
1599 tb_page_addr_t addr;
1600 uint32_t flags;
1602 cpu_get_tb_cpu_state(env, &pc, &cs_base, &flags);
1603 addr = get_page_addr_code(env, pc);
1604 tb_invalidate_phys_range(addr, addr + 1);
1608 #ifndef CONFIG_USER_ONLY
1609 /* in deterministic execution mode, instructions doing device I/Os
1610 must be at the end of the TB */
1611 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
1613 #if defined(TARGET_MIPS) || defined(TARGET_SH4)
1614 CPUArchState *env = cpu->env_ptr;
1615 #endif
1616 TranslationBlock *tb;
1617 uint32_t n, cflags;
1618 target_ulong pc, cs_base;
1619 uint32_t flags;
1621 tb = tb_find_pc(retaddr);
1622 if (!tb) {
1623 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p",
1624 (void *)retaddr);
1626 n = cpu->icount_decr.u16.low + tb->icount;
1627 cpu_restore_state_from_tb(cpu, tb, retaddr);
1628 /* Calculate how many instructions had been executed before the fault
1629 occurred. */
1630 n = n - cpu->icount_decr.u16.low;
1631 /* Generate a new TB ending on the I/O insn. */
1632 n++;
1633 /* On MIPS and SH, delay slot instructions can only be restarted if
1634 they were already the first instruction in the TB. If this is not
1635 the first instruction in a TB then re-execute the preceding
1636 branch. */
1637 #if defined(TARGET_MIPS)
1638 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
1639 env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
1640 cpu->icount_decr.u16.low++;
1641 env->hflags &= ~MIPS_HFLAG_BMASK;
1643 #elif defined(TARGET_SH4)
1644 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
1645 && n > 1) {
1646 env->pc -= 2;
1647 cpu->icount_decr.u16.low++;
1648 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
1650 #endif
1651 /* This should never happen. */
1652 if (n > CF_COUNT_MASK) {
1653 cpu_abort(cpu, "TB too big during recompile");
1656 cflags = n | CF_LAST_IO;
1657 pc = tb->pc;
1658 cs_base = tb->cs_base;
1659 flags = tb->flags;
1660 tb_phys_invalidate(tb, -1);
1661 if (tb->cflags & CF_NOCACHE) {
1662 if (tb->orig_tb) {
1663 /* Invalidate original TB if this TB was generated in
1664 * cpu_exec_nocache() */
1665 tb_phys_invalidate(tb->orig_tb, -1);
1667 tb_free(tb);
1669 /* FIXME: In theory this could raise an exception. In practice
1670 we have already translated the block once so it's probably ok. */
1671 tb_gen_code(cpu, pc, cs_base, flags, cflags);
1672 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
1673 the first in the TB) then we end up generating a whole new TB and
1674 repeating the fault, which is horribly inefficient.
1675 Better would be to execute just this insn uncached, or generate a
1676 second new TB. */
1677 cpu_loop_exit_noexc(cpu);
1680 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr)
1682 unsigned int i;
1684 /* Discard jump cache entries for any tb which might potentially
1685 overlap the flushed page. */
1686 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1687 memset(&cpu->tb_jmp_cache[i], 0,
1688 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1690 i = tb_jmp_cache_hash_page(addr);
1691 memset(&cpu->tb_jmp_cache[i], 0,
1692 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1695 static void print_qht_statistics(FILE *f, fprintf_function cpu_fprintf,
1696 struct qht_stats hst)
1698 uint32_t hgram_opts;
1699 size_t hgram_bins;
1700 char *hgram;
1702 if (!hst.head_buckets) {
1703 return;
1705 cpu_fprintf(f, "TB hash buckets %zu/%zu (%0.2f%% head buckets used)\n",
1706 hst.used_head_buckets, hst.head_buckets,
1707 (double)hst.used_head_buckets / hst.head_buckets * 100);
1709 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
1710 hgram_opts |= QDIST_PR_100X | QDIST_PR_PERCENT;
1711 if (qdist_xmax(&hst.occupancy) - qdist_xmin(&hst.occupancy) == 1) {
1712 hgram_opts |= QDIST_PR_NODECIMAL;
1714 hgram = qdist_pr(&hst.occupancy, 10, hgram_opts);
1715 cpu_fprintf(f, "TB hash occupancy %0.2f%% avg chain occ. Histogram: %s\n",
1716 qdist_avg(&hst.occupancy) * 100, hgram);
1717 g_free(hgram);
1719 hgram_opts = QDIST_PR_BORDER | QDIST_PR_LABELS;
1720 hgram_bins = qdist_xmax(&hst.chain) - qdist_xmin(&hst.chain);
1721 if (hgram_bins > 10) {
1722 hgram_bins = 10;
1723 } else {
1724 hgram_bins = 0;
1725 hgram_opts |= QDIST_PR_NODECIMAL | QDIST_PR_NOBINRANGE;
1727 hgram = qdist_pr(&hst.chain, hgram_bins, hgram_opts);
1728 cpu_fprintf(f, "TB hash avg chain %0.3f buckets. Histogram: %s\n",
1729 qdist_avg(&hst.chain), hgram);
1730 g_free(hgram);
1733 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
1735 int i, target_code_size, max_target_code_size;
1736 int direct_jmp_count, direct_jmp2_count, cross_page;
1737 TranslationBlock *tb;
1738 struct qht_stats hst;
1740 target_code_size = 0;
1741 max_target_code_size = 0;
1742 cross_page = 0;
1743 direct_jmp_count = 0;
1744 direct_jmp2_count = 0;
1745 for (i = 0; i < tcg_ctx.tb_ctx.nb_tbs; i++) {
1746 tb = &tcg_ctx.tb_ctx.tbs[i];
1747 target_code_size += tb->size;
1748 if (tb->size > max_target_code_size) {
1749 max_target_code_size = tb->size;
1751 if (tb->page_addr[1] != -1) {
1752 cross_page++;
1754 if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
1755 direct_jmp_count++;
1756 if (tb->jmp_reset_offset[1] != TB_JMP_RESET_OFFSET_INVALID) {
1757 direct_jmp2_count++;
1761 /* XXX: avoid using doubles ? */
1762 cpu_fprintf(f, "Translation buffer state:\n");
1763 cpu_fprintf(f, "gen code size %td/%zd\n",
1764 tcg_ctx.code_gen_ptr - tcg_ctx.code_gen_buffer,
1765 tcg_ctx.code_gen_highwater - tcg_ctx.code_gen_buffer);
1766 cpu_fprintf(f, "TB count %d/%d\n",
1767 tcg_ctx.tb_ctx.nb_tbs, tcg_ctx.code_gen_max_blocks);
1768 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
1769 tcg_ctx.tb_ctx.nb_tbs ? target_code_size /
1770 tcg_ctx.tb_ctx.nb_tbs : 0,
1771 max_target_code_size);
1772 cpu_fprintf(f, "TB avg host size %td bytes (expansion ratio: %0.1f)\n",
1773 tcg_ctx.tb_ctx.nb_tbs ? (tcg_ctx.code_gen_ptr -
1774 tcg_ctx.code_gen_buffer) /
1775 tcg_ctx.tb_ctx.nb_tbs : 0,
1776 target_code_size ? (double) (tcg_ctx.code_gen_ptr -
1777 tcg_ctx.code_gen_buffer) /
1778 target_code_size : 0);
1779 cpu_fprintf(f, "cross page TB count %d (%d%%)\n", cross_page,
1780 tcg_ctx.tb_ctx.nb_tbs ? (cross_page * 100) /
1781 tcg_ctx.tb_ctx.nb_tbs : 0);
1782 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
1783 direct_jmp_count,
1784 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp_count * 100) /
1785 tcg_ctx.tb_ctx.nb_tbs : 0,
1786 direct_jmp2_count,
1787 tcg_ctx.tb_ctx.nb_tbs ? (direct_jmp2_count * 100) /
1788 tcg_ctx.tb_ctx.nb_tbs : 0);
1790 qht_statistics_init(&tcg_ctx.tb_ctx.htable, &hst);
1791 print_qht_statistics(f, cpu_fprintf, hst);
1792 qht_statistics_destroy(&hst);
1794 cpu_fprintf(f, "\nStatistics:\n");
1795 cpu_fprintf(f, "TB flush count %u\n",
1796 atomic_read(&tcg_ctx.tb_ctx.tb_flush_count));
1797 cpu_fprintf(f, "TB invalidate count %d\n",
1798 tcg_ctx.tb_ctx.tb_phys_invalidate_count);
1799 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
1800 tcg_dump_info(f, cpu_fprintf);
1803 void dump_opcount_info(FILE *f, fprintf_function cpu_fprintf)
1805 tcg_dump_op_count(f, cpu_fprintf);
1808 #else /* CONFIG_USER_ONLY */
1810 void cpu_interrupt(CPUState *cpu, int mask)
1812 cpu->interrupt_request |= mask;
1813 cpu->tcg_exit_req = 1;
1817 * Walks guest process memory "regions" one by one
1818 * and calls callback function 'fn' for each region.
1820 struct walk_memory_regions_data {
1821 walk_memory_regions_fn fn;
1822 void *priv;
1823 target_ulong start;
1824 int prot;
1827 static int walk_memory_regions_end(struct walk_memory_regions_data *data,
1828 target_ulong end, int new_prot)
1830 if (data->start != -1u) {
1831 int rc = data->fn(data->priv, data->start, end, data->prot);
1832 if (rc != 0) {
1833 return rc;
1837 data->start = (new_prot ? end : -1u);
1838 data->prot = new_prot;
1840 return 0;
1843 static int walk_memory_regions_1(struct walk_memory_regions_data *data,
1844 target_ulong base, int level, void **lp)
1846 target_ulong pa;
1847 int i, rc;
1849 if (*lp == NULL) {
1850 return walk_memory_regions_end(data, base, 0);
1853 if (level == 0) {
1854 PageDesc *pd = *lp;
1856 for (i = 0; i < V_L2_SIZE; ++i) {
1857 int prot = pd[i].flags;
1859 pa = base | (i << TARGET_PAGE_BITS);
1860 if (prot != data->prot) {
1861 rc = walk_memory_regions_end(data, pa, prot);
1862 if (rc != 0) {
1863 return rc;
1867 } else {
1868 void **pp = *lp;
1870 for (i = 0; i < V_L2_SIZE; ++i) {
1871 pa = base | ((target_ulong)i <<
1872 (TARGET_PAGE_BITS + V_L2_BITS * level));
1873 rc = walk_memory_regions_1(data, pa, level - 1, pp + i);
1874 if (rc != 0) {
1875 return rc;
1880 return 0;
1883 int walk_memory_regions(void *priv, walk_memory_regions_fn fn)
1885 struct walk_memory_regions_data data;
1886 uintptr_t i;
1888 data.fn = fn;
1889 data.priv = priv;
1890 data.start = -1u;
1891 data.prot = 0;
1893 for (i = 0; i < V_L1_SIZE; i++) {
1894 int rc = walk_memory_regions_1(&data, (target_ulong)i << (V_L1_SHIFT + TARGET_PAGE_BITS),
1895 V_L1_SHIFT / V_L2_BITS - 1, l1_map + i);
1896 if (rc != 0) {
1897 return rc;
1901 return walk_memory_regions_end(&data, 0, 0);
1904 static int dump_region(void *priv, target_ulong start,
1905 target_ulong end, unsigned long prot)
1907 FILE *f = (FILE *)priv;
1909 (void) fprintf(f, TARGET_FMT_lx"-"TARGET_FMT_lx
1910 " "TARGET_FMT_lx" %c%c%c\n",
1911 start, end, end - start,
1912 ((prot & PAGE_READ) ? 'r' : '-'),
1913 ((prot & PAGE_WRITE) ? 'w' : '-'),
1914 ((prot & PAGE_EXEC) ? 'x' : '-'));
1916 return 0;
1919 /* dump memory mappings */
1920 void page_dump(FILE *f)
1922 const int length = sizeof(target_ulong) * 2;
1923 (void) fprintf(f, "%-*s %-*s %-*s %s\n",
1924 length, "start", length, "end", length, "size", "prot");
1925 walk_memory_regions(f, dump_region);
1928 int page_get_flags(target_ulong address)
1930 PageDesc *p;
1932 p = page_find(address >> TARGET_PAGE_BITS);
1933 if (!p) {
1934 return 0;
1936 return p->flags;
1939 /* Modify the flags of a page and invalidate the code if necessary.
1940 The flag PAGE_WRITE_ORG is positioned automatically depending
1941 on PAGE_WRITE. The mmap_lock should already be held. */
1942 void page_set_flags(target_ulong start, target_ulong end, int flags)
1944 target_ulong addr, len;
1946 /* This function should never be called with addresses outside the
1947 guest address space. If this assert fires, it probably indicates
1948 a missing call to h2g_valid. */
1949 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1950 assert(end < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1951 #endif
1952 assert(start < end);
1954 start = start & TARGET_PAGE_MASK;
1955 end = TARGET_PAGE_ALIGN(end);
1957 if (flags & PAGE_WRITE) {
1958 flags |= PAGE_WRITE_ORG;
1961 for (addr = start, len = end - start;
1962 len != 0;
1963 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
1964 PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
1966 /* If the write protection bit is set, then we invalidate
1967 the code inside. */
1968 if (!(p->flags & PAGE_WRITE) &&
1969 (flags & PAGE_WRITE) &&
1970 p->first_tb) {
1971 tb_invalidate_phys_page(addr, 0);
1973 p->flags = flags;
1977 int page_check_range(target_ulong start, target_ulong len, int flags)
1979 PageDesc *p;
1980 target_ulong end;
1981 target_ulong addr;
1983 /* This function should never be called with addresses outside the
1984 guest address space. If this assert fires, it probably indicates
1985 a missing call to h2g_valid. */
1986 #if TARGET_ABI_BITS > L1_MAP_ADDR_SPACE_BITS
1987 assert(start < ((target_ulong)1 << L1_MAP_ADDR_SPACE_BITS));
1988 #endif
1990 if (len == 0) {
1991 return 0;
1993 if (start + len - 1 < start) {
1994 /* We've wrapped around. */
1995 return -1;
1998 /* must do before we loose bits in the next step */
1999 end = TARGET_PAGE_ALIGN(start + len);
2000 start = start & TARGET_PAGE_MASK;
2002 for (addr = start, len = end - start;
2003 len != 0;
2004 len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
2005 p = page_find(addr >> TARGET_PAGE_BITS);
2006 if (!p) {
2007 return -1;
2009 if (!(p->flags & PAGE_VALID)) {
2010 return -1;
2013 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ)) {
2014 return -1;
2016 if (flags & PAGE_WRITE) {
2017 if (!(p->flags & PAGE_WRITE_ORG)) {
2018 return -1;
2020 /* unprotect the page if it was put read-only because it
2021 contains translated code */
2022 if (!(p->flags & PAGE_WRITE)) {
2023 if (!page_unprotect(addr, 0)) {
2024 return -1;
2029 return 0;
2032 /* called from signal handler: invalidate the code and unprotect the
2033 * page. Return 0 if the fault was not handled, 1 if it was handled,
2034 * and 2 if it was handled but the caller must cause the TB to be
2035 * immediately exited. (We can only return 2 if the 'pc' argument is
2036 * non-zero.)
2038 int page_unprotect(target_ulong address, uintptr_t pc)
2040 unsigned int prot;
2041 bool current_tb_invalidated;
2042 PageDesc *p;
2043 target_ulong host_start, host_end, addr;
2045 /* Technically this isn't safe inside a signal handler. However we
2046 know this only ever happens in a synchronous SEGV handler, so in
2047 practice it seems to be ok. */
2048 mmap_lock();
2050 p = page_find(address >> TARGET_PAGE_BITS);
2051 if (!p) {
2052 mmap_unlock();
2053 return 0;
2056 /* if the page was really writable, then we change its
2057 protection back to writable */
2058 if ((p->flags & PAGE_WRITE_ORG) && !(p->flags & PAGE_WRITE)) {
2059 host_start = address & qemu_host_page_mask;
2060 host_end = host_start + qemu_host_page_size;
2062 prot = 0;
2063 current_tb_invalidated = false;
2064 for (addr = host_start ; addr < host_end ; addr += TARGET_PAGE_SIZE) {
2065 p = page_find(addr >> TARGET_PAGE_BITS);
2066 p->flags |= PAGE_WRITE;
2067 prot |= p->flags;
2069 /* and since the content will be modified, we must invalidate
2070 the corresponding translated code. */
2071 current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
2072 #ifdef DEBUG_TB_CHECK
2073 tb_invalidate_check(addr);
2074 #endif
2076 mprotect((void *)g2h(host_start), qemu_host_page_size,
2077 prot & PAGE_BITS);
2079 mmap_unlock();
2080 /* If current TB was invalidated return to main loop */
2081 return current_tb_invalidated ? 2 : 1;
2083 mmap_unlock();
2084 return 0;
2086 #endif /* CONFIG_USER_ONLY */