2 * emulator main execution loop
4 * Copyright (c) 2003-2005 Fabrice Bellard
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20 #include "qemu/osdep.h"
21 #include "qemu/qemu-print.h"
22 #include "qapi/error.h"
23 #include "qapi/type-helpers.h"
24 #include "hw/core/tcg-cpu-ops.h"
26 #include "disas/disas.h"
27 #include "exec/exec-all.h"
29 #include "qemu/atomic.h"
32 #include "qemu/main-loop.h"
33 #include "sysemu/cpus.h"
34 #include "exec/cpu-all.h"
35 #include "sysemu/cpu-timers.h"
36 #include "exec/replay-core.h"
37 #include "sysemu/tcg.h"
38 #include "exec/helper-proto-common.h"
39 #include "tb-jmp-cache.h"
41 #include "tb-context.h"
42 #include "internal-common.h"
43 #include "internal-target.h"
45 /* -icount align implementation. */
47 typedef struct SyncClocks
{
49 int64_t last_cpu_icount
;
50 int64_t realtime_clock
;
53 #if !defined(CONFIG_USER_ONLY)
54 /* Allow the guest to have a max 3ms advance.
55 * The difference between the 2 clocks could therefore
58 #define VM_CLOCK_ADVANCE 3000000
59 #define THRESHOLD_REDUCE 1.5
60 #define MAX_DELAY_PRINT_RATE 2000000000LL
61 #define MAX_NB_PRINTS 100
66 static void align_clocks(SyncClocks
*sc
, CPUState
*cpu
)
70 if (!icount_align_option
) {
74 cpu_icount
= cpu
->icount_extra
+ cpu
->neg
.icount_decr
.u16
.low
;
75 sc
->diff_clk
+= icount_to_ns(sc
->last_cpu_icount
- cpu_icount
);
76 sc
->last_cpu_icount
= cpu_icount
;
78 if (sc
->diff_clk
> VM_CLOCK_ADVANCE
) {
80 struct timespec sleep_delay
, rem_delay
;
81 sleep_delay
.tv_sec
= sc
->diff_clk
/ 1000000000LL;
82 sleep_delay
.tv_nsec
= sc
->diff_clk
% 1000000000LL;
83 if (nanosleep(&sleep_delay
, &rem_delay
) < 0) {
84 sc
->diff_clk
= rem_delay
.tv_sec
* 1000000000LL + rem_delay
.tv_nsec
;
89 Sleep(sc
->diff_clk
/ SCALE_MS
);
95 static void print_delay(const SyncClocks
*sc
)
97 static float threshold_delay
;
98 static int64_t last_realtime_clock
;
101 if (icount_align_option
&&
102 sc
->realtime_clock
- last_realtime_clock
>= MAX_DELAY_PRINT_RATE
&&
103 nb_prints
< MAX_NB_PRINTS
) {
104 if ((-sc
->diff_clk
/ (float)1000000000LL > threshold_delay
) ||
105 (-sc
->diff_clk
/ (float)1000000000LL <
106 (threshold_delay
- THRESHOLD_REDUCE
))) {
107 threshold_delay
= (-sc
->diff_clk
/ 1000000000LL) + 1;
108 qemu_printf("Warning: The guest is now late by %.1f to %.1f seconds\n",
112 last_realtime_clock
= sc
->realtime_clock
;
117 static void init_delay_params(SyncClocks
*sc
, CPUState
*cpu
)
119 if (!icount_align_option
) {
122 sc
->realtime_clock
= qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT
);
123 sc
->diff_clk
= qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL
) - sc
->realtime_clock
;
125 = cpu
->icount_extra
+ cpu
->neg
.icount_decr
.u16
.low
;
126 if (sc
->diff_clk
< max_delay
) {
127 max_delay
= sc
->diff_clk
;
129 if (sc
->diff_clk
> max_advance
) {
130 max_advance
= sc
->diff_clk
;
133 /* Print every 2s max if the guest is late. We limit the number
134 of printed messages to NB_PRINT_MAX(currently 100) */
138 static void align_clocks(SyncClocks
*sc
, const CPUState
*cpu
)
142 static void init_delay_params(SyncClocks
*sc
, const CPUState
*cpu
)
145 #endif /* CONFIG USER ONLY */
147 uint32_t curr_cflags(CPUState
*cpu
)
149 uint32_t cflags
= cpu
->tcg_cflags
;
152 * Record gdb single-step. We should be exiting the TB by raising
153 * EXCP_DEBUG, but to simplify other tests, disable chaining too.
155 * For singlestep and -d nochain, suppress goto_tb so that
156 * we can log -d cpu,exec after every TB.
158 if (unlikely(cpu
->singlestep_enabled
)) {
159 cflags
|= CF_NO_GOTO_TB
| CF_NO_GOTO_PTR
| CF_SINGLE_STEP
| 1;
160 } else if (qatomic_read(&one_insn_per_tb
)) {
161 cflags
|= CF_NO_GOTO_TB
| 1;
162 } else if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN
)) {
163 cflags
|= CF_NO_GOTO_TB
;
173 tb_page_addr_t page_addr0
;
178 static bool tb_lookup_cmp(const void *p
, const void *d
)
180 const TranslationBlock
*tb
= p
;
181 const struct tb_desc
*desc
= d
;
183 if ((tb_cflags(tb
) & CF_PCREL
|| tb
->pc
== desc
->pc
) &&
184 tb_page_addr0(tb
) == desc
->page_addr0
&&
185 tb
->cs_base
== desc
->cs_base
&&
186 tb
->flags
== desc
->flags
&&
187 tb_cflags(tb
) == desc
->cflags
) {
188 /* check next page if needed */
189 tb_page_addr_t tb_phys_page1
= tb_page_addr1(tb
);
190 if (tb_phys_page1
== -1) {
193 tb_page_addr_t phys_page1
;
197 * We know that the first page matched, and an otherwise valid TB
198 * encountered an incomplete instruction at the end of that page,
199 * therefore we know that generating a new TB from the current PC
200 * must also require reading from the next page -- even if the
201 * second pages do not match, and therefore the resulting insn
202 * is different for the new TB. Therefore any exception raised
203 * here by the faulting lookup is not premature.
205 virt_page1
= TARGET_PAGE_ALIGN(desc
->pc
);
206 phys_page1
= get_page_addr_code(desc
->env
, virt_page1
);
207 if (tb_phys_page1
== phys_page1
) {
215 static TranslationBlock
*tb_htable_lookup(CPUState
*cpu
, vaddr pc
,
216 uint64_t cs_base
, uint32_t flags
,
219 tb_page_addr_t phys_pc
;
223 desc
.env
= cpu_env(cpu
);
224 desc
.cs_base
= cs_base
;
226 desc
.cflags
= cflags
;
228 phys_pc
= get_page_addr_code(desc
.env
, pc
);
232 desc
.page_addr0
= phys_pc
;
233 h
= tb_hash_func(phys_pc
, (cflags
& CF_PCREL
? 0 : pc
),
234 flags
, cs_base
, cflags
);
235 return qht_lookup_custom(&tb_ctx
.htable
, &desc
, h
, tb_lookup_cmp
);
238 /* Might cause an exception, so have a longjmp destination ready */
239 static inline TranslationBlock
*tb_lookup(CPUState
*cpu
, vaddr pc
,
240 uint64_t cs_base
, uint32_t flags
,
243 TranslationBlock
*tb
;
247 /* we should never be trying to look up an INVALID tb */
248 tcg_debug_assert(!(cflags
& CF_INVALID
));
250 hash
= tb_jmp_cache_hash_func(pc
);
251 jc
= cpu
->tb_jmp_cache
;
253 tb
= qatomic_read(&jc
->array
[hash
].tb
);
255 jc
->array
[hash
].pc
== pc
&&
256 tb
->cs_base
== cs_base
&&
257 tb
->flags
== flags
&&
258 tb_cflags(tb
) == cflags
)) {
262 tb
= tb_htable_lookup(cpu
, pc
, cs_base
, flags
, cflags
);
267 jc
->array
[hash
].pc
= pc
;
268 qatomic_set(&jc
->array
[hash
].tb
, tb
);
272 * As long as tb is not NULL, the contents are consistent. Therefore,
273 * the virtual PC has to match for non-CF_PCREL translations.
275 assert((tb_cflags(tb
) & CF_PCREL
) || tb
->pc
== pc
);
279 static void log_cpu_exec(vaddr pc
, CPUState
*cpu
,
280 const TranslationBlock
*tb
)
282 if (qemu_log_in_addr_range(pc
)) {
283 qemu_log_mask(CPU_LOG_EXEC
,
284 "Trace %d: %p [%08" PRIx64
285 "/%016" VADDR_PRIx
"/%08x/%08x] %s\n",
286 cpu
->cpu_index
, tb
->tc
.ptr
, tb
->cs_base
, pc
,
287 tb
->flags
, tb
->cflags
, lookup_symbol(pc
));
289 if (qemu_loglevel_mask(CPU_LOG_TB_CPU
)) {
290 FILE *logfile
= qemu_log_trylock();
294 if (qemu_loglevel_mask(CPU_LOG_TB_FPU
)) {
295 flags
|= CPU_DUMP_FPU
;
297 #if defined(TARGET_I386)
298 flags
|= CPU_DUMP_CCOP
;
300 if (qemu_loglevel_mask(CPU_LOG_TB_VPU
)) {
301 flags
|= CPU_DUMP_VPU
;
303 cpu_dump_state(cpu
, logfile
, flags
);
304 qemu_log_unlock(logfile
);
310 static bool check_for_breakpoints_slow(CPUState
*cpu
, vaddr pc
,
314 bool match_page
= false;
317 * Singlestep overrides breakpoints.
318 * This requirement is visible in the record-replay tests, where
319 * we would fail to make forward progress in reverse-continue.
321 * TODO: gdb singlestep should only override gdb breakpoints,
322 * so that one could (gdb) singlestep into the guest kernel's
323 * architectural breakpoint handler.
325 if (cpu
->singlestep_enabled
) {
329 QTAILQ_FOREACH(bp
, &cpu
->breakpoints
, entry
) {
331 * If we have an exact pc match, trigger the breakpoint.
332 * Otherwise, note matches within the page.
335 bool match_bp
= false;
337 if (bp
->flags
& BP_GDB
) {
339 } else if (bp
->flags
& BP_CPU
) {
340 #ifdef CONFIG_USER_ONLY
341 g_assert_not_reached();
343 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
344 assert(tcg_ops
->debug_check_breakpoint
);
345 match_bp
= tcg_ops
->debug_check_breakpoint(cpu
);
350 cpu
->exception_index
= EXCP_DEBUG
;
353 } else if (((pc
^ bp
->pc
) & TARGET_PAGE_MASK
) == 0) {
359 * Within the same page as a breakpoint, single-step,
360 * returning to helper_lookup_tb_ptr after each insn looking
361 * for the actual breakpoint.
363 * TODO: Perhaps better to record all of the TBs associated
364 * with a given virtual page that contains a breakpoint, and
365 * then invalidate them when a new overlapping breakpoint is
366 * set on the page. Non-overlapping TBs would not be
367 * invalidated, nor would any TB need to be invalidated as
368 * breakpoints are removed.
371 *cflags
= (*cflags
& ~CF_COUNT_MASK
) | CF_NO_GOTO_TB
| 1;
376 static inline bool check_for_breakpoints(CPUState
*cpu
, vaddr pc
,
379 return unlikely(!QTAILQ_EMPTY(&cpu
->breakpoints
)) &&
380 check_for_breakpoints_slow(cpu
, pc
, cflags
);
384 * helper_lookup_tb_ptr: quick check for next tb
385 * @env: current cpu state
387 * Look for an existing TB matching the current cpu state.
388 * If found, return the code pointer. If not found, return
389 * the tcg epilogue so that we return into cpu_tb_exec.
391 const void *HELPER(lookup_tb_ptr
)(CPUArchState
*env
)
393 CPUState
*cpu
= env_cpu(env
);
394 TranslationBlock
*tb
;
397 uint32_t flags
, cflags
;
400 * By definition we've just finished a TB, so I/O is OK.
401 * Avoid the possibility of calling cpu_io_recompile() if
402 * a page table walk triggered by tb_lookup() calling
403 * probe_access_internal() happens to touch an MMIO device.
404 * The next TB, if we chain to it, will clear the flag again.
406 cpu
->neg
.can_do_io
= true;
407 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
409 cflags
= curr_cflags(cpu
);
410 if (check_for_breakpoints(cpu
, pc
, &cflags
)) {
414 tb
= tb_lookup(cpu
, pc
, cs_base
, flags
, cflags
);
416 return tcg_code_gen_epilogue
;
419 if (qemu_loglevel_mask(CPU_LOG_TB_CPU
| CPU_LOG_EXEC
)) {
420 log_cpu_exec(pc
, cpu
, tb
);
426 /* Execute a TB, and fix up the CPU state afterwards if necessary */
428 * Disable CFI checks.
429 * TCG creates binary blobs at runtime, with the transformed code.
430 * A TB is a blob of binary code, created at runtime and called with an
431 * indirect function call. Since such function did not exist at compile time,
432 * the CFI runtime has no way to verify its signature and would fail.
433 * TCG is not considered a security-sensitive part of QEMU so this does not
434 * affect the impact of CFI in environment with high security requirements
436 static inline TranslationBlock
* QEMU_DISABLE_CFI
437 cpu_tb_exec(CPUState
*cpu
, TranslationBlock
*itb
, int *tb_exit
)
439 CPUArchState
*env
= cpu_env(cpu
);
441 TranslationBlock
*last_tb
;
442 const void *tb_ptr
= itb
->tc
.ptr
;
444 if (qemu_loglevel_mask(CPU_LOG_TB_CPU
| CPU_LOG_EXEC
)) {
445 log_cpu_exec(log_pc(cpu
, itb
), cpu
, itb
);
448 qemu_thread_jit_execute();
449 ret
= tcg_qemu_tb_exec(env
, tb_ptr
);
450 cpu
->neg
.can_do_io
= true;
451 qemu_plugin_disable_mem_helpers(cpu
);
453 * TODO: Delay swapping back to the read-write region of the TB
454 * until we actually need to modify the TB. The read-only copy,
455 * coming from the rx region, shares the same host TLB entry as
456 * the code that executed the exit_tb opcode that arrived here.
457 * If we insist on touching both the RX and the RW pages, we
458 * double the host TLB pressure.
460 last_tb
= tcg_splitwx_to_rw((void *)(ret
& ~TB_EXIT_MASK
));
461 *tb_exit
= ret
& TB_EXIT_MASK
;
463 trace_exec_tb_exit(last_tb
, *tb_exit
);
465 if (*tb_exit
> TB_EXIT_IDX1
) {
466 /* We didn't start executing this TB (eg because the instruction
467 * counter hit zero); we must restore the guest PC to the address
468 * of the start of the TB.
470 CPUClass
*cc
= cpu
->cc
;
471 const TCGCPUOps
*tcg_ops
= cc
->tcg_ops
;
473 if (tcg_ops
->synchronize_from_tb
) {
474 tcg_ops
->synchronize_from_tb(cpu
, last_tb
);
476 tcg_debug_assert(!(tb_cflags(last_tb
) & CF_PCREL
));
478 cc
->set_pc(cpu
, last_tb
->pc
);
480 if (qemu_loglevel_mask(CPU_LOG_EXEC
)) {
481 vaddr pc
= log_pc(cpu
, last_tb
);
482 if (qemu_log_in_addr_range(pc
)) {
483 qemu_log("Stopped execution of TB chain before %p [%016"
485 last_tb
->tc
.ptr
, pc
, lookup_symbol(pc
));
491 * If gdb single-step, and we haven't raised another exception,
492 * raise a debug exception. Single-step with another exception
493 * is handled in cpu_handle_exception.
495 if (unlikely(cpu
->singlestep_enabled
) && cpu
->exception_index
== -1) {
496 cpu
->exception_index
= EXCP_DEBUG
;
504 static void cpu_exec_enter(CPUState
*cpu
)
506 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
508 if (tcg_ops
->cpu_exec_enter
) {
509 tcg_ops
->cpu_exec_enter(cpu
);
513 static void cpu_exec_exit(CPUState
*cpu
)
515 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
517 if (tcg_ops
->cpu_exec_exit
) {
518 tcg_ops
->cpu_exec_exit(cpu
);
522 static void cpu_exec_longjmp_cleanup(CPUState
*cpu
)
524 /* Non-buggy compilers preserve this; assert the correct value. */
525 g_assert(cpu
== current_cpu
);
527 #ifdef CONFIG_USER_ONLY
528 clear_helper_retaddr();
529 if (have_mmap_lock()) {
534 * For softmmu, a tlb_fill fault during translation will land here,
535 * and we need to release any page locks held. In system mode we
536 * have one tcg_ctx per thread, so we know it was this cpu doing
539 * Alternative 1: Install a cleanup to be called via an exception
540 * handling safe longjmp. It seems plausible that all our hosts
541 * support such a thing. We'd have to properly register unwind info
542 * for the JIT for EH, rather that just for GDB.
544 * Alternative 2: Set and restore cpu->jmp_env in tb_gen_code to
545 * capture the cpu_loop_exit longjmp, perform the cleanup, and
546 * jump again to arrive here.
548 if (tcg_ctx
->gen_tb
) {
549 tb_unlock_pages(tcg_ctx
->gen_tb
);
550 tcg_ctx
->gen_tb
= NULL
;
556 assert_no_pages_locked();
559 void cpu_exec_step_atomic(CPUState
*cpu
)
561 CPUArchState
*env
= cpu_env(cpu
);
562 TranslationBlock
*tb
;
565 uint32_t flags
, cflags
;
568 if (sigsetjmp(cpu
->jmp_env
, 0) == 0) {
570 g_assert(cpu
== current_cpu
);
571 g_assert(!cpu
->running
);
574 cpu_get_tb_cpu_state(env
, &pc
, &cs_base
, &flags
);
576 cflags
= curr_cflags(cpu
);
577 /* Execute in a serial context. */
578 cflags
&= ~CF_PARALLEL
;
579 /* After 1 insn, return and release the exclusive lock. */
580 cflags
|= CF_NO_GOTO_TB
| CF_NO_GOTO_PTR
| 1;
582 * No need to check_for_breakpoints here.
583 * We only arrive in cpu_exec_step_atomic after beginning execution
584 * of an insn that includes an atomic operation we can't handle.
585 * Any breakpoint for this insn will have been recognized earlier.
588 tb
= tb_lookup(cpu
, pc
, cs_base
, flags
, cflags
);
591 tb
= tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
596 /* execute the generated code */
597 trace_exec_tb(tb
, pc
);
598 cpu_tb_exec(cpu
, tb
, &tb_exit
);
601 cpu_exec_longjmp_cleanup(cpu
);
605 * As we start the exclusive region before codegen we must still
606 * be in the region if we longjump out of either the codegen or
609 g_assert(cpu_in_exclusive_context(cpu
));
610 cpu
->running
= false;
614 void tb_set_jmp_target(TranslationBlock
*tb
, int n
, uintptr_t addr
)
617 * Get the rx view of the structure, from which we find the
618 * executable code address, and tb_target_set_jmp_target can
619 * produce a pc-relative displacement to jmp_target_addr[n].
621 const TranslationBlock
*c_tb
= tcg_splitwx_to_rx(tb
);
622 uintptr_t offset
= tb
->jmp_insn_offset
[n
];
623 uintptr_t jmp_rx
= (uintptr_t)tb
->tc
.ptr
+ offset
;
624 uintptr_t jmp_rw
= jmp_rx
- tcg_splitwx_diff
;
626 tb
->jmp_target_addr
[n
] = addr
;
627 tb_target_set_jmp_target(c_tb
, n
, jmp_rx
, jmp_rw
);
630 static inline void tb_add_jump(TranslationBlock
*tb
, int n
,
631 TranslationBlock
*tb_next
)
635 qemu_thread_jit_write();
636 assert(n
< ARRAY_SIZE(tb
->jmp_list_next
));
637 qemu_spin_lock(&tb_next
->jmp_lock
);
639 /* make sure the destination TB is valid */
640 if (tb_next
->cflags
& CF_INVALID
) {
641 goto out_unlock_next
;
643 /* Atomically claim the jump destination slot only if it was NULL */
644 old
= qatomic_cmpxchg(&tb
->jmp_dest
[n
], (uintptr_t)NULL
,
647 goto out_unlock_next
;
650 /* patch the native jump address */
651 tb_set_jmp_target(tb
, n
, (uintptr_t)tb_next
->tc
.ptr
);
653 /* add in TB jmp list */
654 tb
->jmp_list_next
[n
] = tb_next
->jmp_list_head
;
655 tb_next
->jmp_list_head
= (uintptr_t)tb
| n
;
657 qemu_spin_unlock(&tb_next
->jmp_lock
);
659 qemu_log_mask(CPU_LOG_EXEC
, "Linking TBs %p index %d -> %p\n",
660 tb
->tc
.ptr
, n
, tb_next
->tc
.ptr
);
664 qemu_spin_unlock(&tb_next
->jmp_lock
);
668 static inline bool cpu_handle_halt(CPUState
*cpu
)
670 #ifndef CONFIG_USER_ONLY
672 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
674 if (tcg_ops
->cpu_exec_halt
) {
675 tcg_ops
->cpu_exec_halt(cpu
);
677 if (!cpu_has_work(cpu
)) {
683 #endif /* !CONFIG_USER_ONLY */
688 static inline void cpu_handle_debug_exception(CPUState
*cpu
)
690 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
693 if (!cpu
->watchpoint_hit
) {
694 QTAILQ_FOREACH(wp
, &cpu
->watchpoints
, entry
) {
695 wp
->flags
&= ~BP_WATCHPOINT_HIT
;
699 if (tcg_ops
->debug_excp_handler
) {
700 tcg_ops
->debug_excp_handler(cpu
);
704 static inline bool cpu_handle_exception(CPUState
*cpu
, int *ret
)
706 if (cpu
->exception_index
< 0) {
707 #ifndef CONFIG_USER_ONLY
708 if (replay_has_exception()
709 && cpu
->neg
.icount_decr
.u16
.low
+ cpu
->icount_extra
== 0) {
710 /* Execute just one insn to trigger exception pending in the log */
711 cpu
->cflags_next_tb
= (curr_cflags(cpu
) & ~CF_USE_ICOUNT
)
718 if (cpu
->exception_index
>= EXCP_INTERRUPT
) {
719 /* exit request from the cpu execution loop */
720 *ret
= cpu
->exception_index
;
721 if (*ret
== EXCP_DEBUG
) {
722 cpu_handle_debug_exception(cpu
);
724 cpu
->exception_index
= -1;
728 #if defined(CONFIG_USER_ONLY)
730 * If user mode only, we simulate a fake exception which will be
731 * handled outside the cpu execution loop.
733 #if defined(TARGET_I386)
734 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
735 tcg_ops
->fake_user_interrupt(cpu
);
736 #endif /* TARGET_I386 */
737 *ret
= cpu
->exception_index
;
738 cpu
->exception_index
= -1;
741 if (replay_exception()) {
742 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
745 tcg_ops
->do_interrupt(cpu
);
747 cpu
->exception_index
= -1;
749 if (unlikely(cpu
->singlestep_enabled
)) {
751 * After processing the exception, ensure an EXCP_DEBUG is
752 * raised when single-stepping so that GDB doesn't miss the
756 cpu_handle_debug_exception(cpu
);
759 } else if (!replay_has_interrupt()) {
760 /* give a chance to iothread in replay mode */
761 *ret
= EXCP_INTERRUPT
;
769 static inline bool icount_exit_request(CPUState
*cpu
)
771 if (!icount_enabled()) {
774 if (cpu
->cflags_next_tb
!= -1 && !(cpu
->cflags_next_tb
& CF_USE_ICOUNT
)) {
777 return cpu
->neg
.icount_decr
.u16
.low
+ cpu
->icount_extra
== 0;
780 static inline bool cpu_handle_interrupt(CPUState
*cpu
,
781 TranslationBlock
**last_tb
)
784 * If we have requested custom cflags with CF_NOIRQ we should
785 * skip checking here. Any pending interrupts will get picked up
786 * by the next TB we execute under normal cflags.
788 if (cpu
->cflags_next_tb
!= -1 && cpu
->cflags_next_tb
& CF_NOIRQ
) {
792 /* Clear the interrupt flag now since we're processing
793 * cpu->interrupt_request and cpu->exit_request.
794 * Ensure zeroing happens before reading cpu->exit_request or
795 * cpu->interrupt_request (see also smp_wmb in cpu_exit())
797 qatomic_set_mb(&cpu
->neg
.icount_decr
.u16
.high
, 0);
799 if (unlikely(qatomic_read(&cpu
->interrupt_request
))) {
800 int interrupt_request
;
802 interrupt_request
= cpu
->interrupt_request
;
803 if (unlikely(cpu
->singlestep_enabled
& SSTEP_NOIRQ
)) {
804 /* Mask out external interrupts for this step. */
805 interrupt_request
&= ~CPU_INTERRUPT_SSTEP_MASK
;
807 if (interrupt_request
& CPU_INTERRUPT_DEBUG
) {
808 cpu
->interrupt_request
&= ~CPU_INTERRUPT_DEBUG
;
809 cpu
->exception_index
= EXCP_DEBUG
;
813 #if !defined(CONFIG_USER_ONLY)
814 if (replay_mode
== REPLAY_MODE_PLAY
&& !replay_has_interrupt()) {
816 } else if (interrupt_request
& CPU_INTERRUPT_HALT
) {
818 cpu
->interrupt_request
&= ~CPU_INTERRUPT_HALT
;
820 cpu
->exception_index
= EXCP_HLT
;
824 #if defined(TARGET_I386)
825 else if (interrupt_request
& CPU_INTERRUPT_INIT
) {
826 X86CPU
*x86_cpu
= X86_CPU(cpu
);
827 CPUArchState
*env
= &x86_cpu
->env
;
829 cpu_svm_check_intercept_param(env
, SVM_EXIT_INIT
, 0, 0);
830 do_cpu_init(x86_cpu
);
831 cpu
->exception_index
= EXCP_HALTED
;
836 else if (interrupt_request
& CPU_INTERRUPT_RESET
) {
842 #endif /* !TARGET_I386 */
843 /* The target hook has 3 exit conditions:
844 False when the interrupt isn't processed,
845 True when it is, and we should restart on a new TB,
846 and via longjmp via cpu_loop_exit. */
848 const TCGCPUOps
*tcg_ops
= cpu
->cc
->tcg_ops
;
850 if (tcg_ops
->cpu_exec_interrupt
&&
851 tcg_ops
->cpu_exec_interrupt(cpu
, interrupt_request
)) {
852 if (!tcg_ops
->need_replay_interrupt
||
853 tcg_ops
->need_replay_interrupt(interrupt_request
)) {
857 * After processing the interrupt, ensure an EXCP_DEBUG is
858 * raised when single-stepping so that GDB doesn't miss the
861 if (unlikely(cpu
->singlestep_enabled
)) {
862 cpu
->exception_index
= EXCP_DEBUG
;
866 cpu
->exception_index
= -1;
869 /* The target hook may have updated the 'cpu->interrupt_request';
870 * reload the 'interrupt_request' value */
871 interrupt_request
= cpu
->interrupt_request
;
873 #endif /* !CONFIG_USER_ONLY */
874 if (interrupt_request
& CPU_INTERRUPT_EXITTB
) {
875 cpu
->interrupt_request
&= ~CPU_INTERRUPT_EXITTB
;
876 /* ensure that no TB jump will be modified as
877 the program flow was changed */
881 /* If we exit via cpu_loop_exit/longjmp it is reset in cpu_exec */
885 /* Finally, check if we need to exit to the main loop. */
886 if (unlikely(qatomic_read(&cpu
->exit_request
)) || icount_exit_request(cpu
)) {
887 qatomic_set(&cpu
->exit_request
, 0);
888 if (cpu
->exception_index
== -1) {
889 cpu
->exception_index
= EXCP_INTERRUPT
;
897 static inline void cpu_loop_exec_tb(CPUState
*cpu
, TranslationBlock
*tb
,
898 vaddr pc
, TranslationBlock
**last_tb
,
903 trace_exec_tb(tb
, pc
);
904 tb
= cpu_tb_exec(cpu
, tb
, tb_exit
);
905 if (*tb_exit
!= TB_EXIT_REQUESTED
) {
911 insns_left
= qatomic_read(&cpu
->neg
.icount_decr
.u32
);
912 if (insns_left
< 0) {
913 /* Something asked us to stop executing chained TBs; just
914 * continue round the main loop. Whatever requested the exit
915 * will also have set something else (eg exit_request or
916 * interrupt_request) which will be handled by
917 * cpu_handle_interrupt. cpu_handle_interrupt will also
918 * clear cpu->icount_decr.u16.high.
923 /* Instruction counter expired. */
924 assert(icount_enabled());
925 #ifndef CONFIG_USER_ONLY
926 /* Ensure global icount has gone forward */
928 /* Refill decrementer and continue execution. */
929 insns_left
= MIN(0xffff, cpu
->icount_budget
);
930 cpu
->neg
.icount_decr
.u16
.low
= insns_left
;
931 cpu
->icount_extra
= cpu
->icount_budget
- insns_left
;
934 * If the next tb has more instructions than we have left to
935 * execute we need to ensure we find/generate a TB with exactly
936 * insns_left instructions in it.
938 if (insns_left
> 0 && insns_left
< tb
->icount
) {
939 assert(insns_left
<= CF_COUNT_MASK
);
940 assert(cpu
->icount_extra
== 0);
941 cpu
->cflags_next_tb
= (tb
->cflags
& ~CF_COUNT_MASK
) | insns_left
;
946 /* main execution loop */
948 static int __attribute__((noinline
))
949 cpu_exec_loop(CPUState
*cpu
, SyncClocks
*sc
)
953 /* if an exception is pending, we execute it here */
954 while (!cpu_handle_exception(cpu
, &ret
)) {
955 TranslationBlock
*last_tb
= NULL
;
958 while (!cpu_handle_interrupt(cpu
, &last_tb
)) {
959 TranslationBlock
*tb
;
962 uint32_t flags
, cflags
;
964 cpu_get_tb_cpu_state(cpu_env(cpu
), &pc
, &cs_base
, &flags
);
967 * When requested, use an exact setting for cflags for the next
968 * execution. This is used for icount, precise smc, and stop-
969 * after-access watchpoints. Since this request should never
970 * have CF_INVALID set, -1 is a convenient invalid value that
971 * does not require tcg headers for cpu_common_reset.
973 cflags
= cpu
->cflags_next_tb
;
975 cflags
= curr_cflags(cpu
);
977 cpu
->cflags_next_tb
= -1;
980 if (check_for_breakpoints(cpu
, pc
, &cflags
)) {
984 tb
= tb_lookup(cpu
, pc
, cs_base
, flags
, cflags
);
990 tb
= tb_gen_code(cpu
, pc
, cs_base
, flags
, cflags
);
994 * We add the TB in the virtual pc hash table
995 * for the fast lookup
997 h
= tb_jmp_cache_hash_func(pc
);
998 jc
= cpu
->tb_jmp_cache
;
999 jc
->array
[h
].pc
= pc
;
1000 qatomic_set(&jc
->array
[h
].tb
, tb
);
1003 #ifndef CONFIG_USER_ONLY
1005 * We don't take care of direct jumps when address mapping
1006 * changes in system emulation. So it's not safe to make a
1007 * direct jump to a TB spanning two pages because the mapping
1008 * for the second page can change.
1010 if (tb_page_addr1(tb
) != -1) {
1014 /* See if we can patch the calling TB. */
1016 tb_add_jump(last_tb
, tb_exit
, tb
);
1019 cpu_loop_exec_tb(cpu
, tb
, pc
, &last_tb
, &tb_exit
);
1021 /* Try to align the host and virtual clocks
1022 if the guest is in advance */
1023 align_clocks(sc
, cpu
);
1029 static int cpu_exec_setjmp(CPUState
*cpu
, SyncClocks
*sc
)
1031 /* Prepare setjmp context for exception handling. */
1032 if (unlikely(sigsetjmp(cpu
->jmp_env
, 0) != 0)) {
1033 cpu_exec_longjmp_cleanup(cpu
);
1036 return cpu_exec_loop(cpu
, sc
);
1039 int cpu_exec(CPUState
*cpu
)
1042 SyncClocks sc
= { 0 };
1044 /* replay_interrupt may need current_cpu */
1047 if (cpu_handle_halt(cpu
)) {
1051 RCU_READ_LOCK_GUARD();
1052 cpu_exec_enter(cpu
);
1055 * Calculate difference between guest clock and host clock.
1056 * This delay includes the delay of the last cycle, so
1057 * what we have to do is sleep until it is 0. As for the
1058 * advance/delay we gain here, we try to fix it next time.
1060 init_delay_params(&sc
, cpu
);
1062 ret
= cpu_exec_setjmp(cpu
, &sc
);
1068 bool tcg_exec_realizefn(CPUState
*cpu
, Error
**errp
)
1070 static bool tcg_target_initialized
;
1072 if (!tcg_target_initialized
) {
1073 cpu
->cc
->tcg_ops
->initialize();
1074 tcg_target_initialized
= true;
1077 cpu
->tb_jmp_cache
= g_new0(CPUJumpCache
, 1);
1079 #ifndef CONFIG_USER_ONLY
1080 tcg_iommu_init_notifier_list(cpu
);
1081 #endif /* !CONFIG_USER_ONLY */
1082 /* qemu_plugin_vcpu_init_hook delayed until cpu_index assigned. */
1087 /* undo the initializations in reverse order */
1088 void tcg_exec_unrealizefn(CPUState
*cpu
)
1090 #ifndef CONFIG_USER_ONLY
1091 tcg_iommu_free_notifier_list(cpu
);
1092 #endif /* !CONFIG_USER_ONLY */
1095 g_free_rcu(cpu
->tb_jmp_cache
, rcu
);