include/exec/exec-all.h

   1 /*
   2  * internal execution defines for qemu
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #ifndef EXEC_ALL_H
  21 #define EXEC_ALL_H
  22
  23 #include "qemu-common.h"
  24 #include "exec/tb-context.h"
  25 #include "sysemu/cpus.h"
  26
  27 /* allow to see translation results - the slowdown should be negligible, so we leave it */
  28 #define DEBUG_DISAS
  29
  30 /* Page tracking code uses ram addresses in system mode, and virtual
  31    addresses in userspace mode.  Define tb_page_addr_t to be an appropriate
  32    type.  */
  33 #if defined(CONFIG_USER_ONLY)
  34 typedef abi_ulong tb_page_addr_t;
  35 #define TB_PAGE_ADDR_FMT TARGET_ABI_FMT_lx
  36 #else
  37 typedef ram_addr_t tb_page_addr_t;
  38 #define TB_PAGE_ADDR_FMT RAM_ADDR_FMT
  39 #endif
  40
  41 #include "qemu/log.h"
  42
  43 void gen_intermediate_code(CPUState *cpu, struct TranslationBlock *tb);
  44 void restore_state_to_opc(CPUArchState *env, struct TranslationBlock *tb,
  45                           target_ulong *data);
  46
  47 /* Get a backtrace for the guest code. */
  48 const char *qemu_sprint_backtrace(char *buffer, size_t length);
  49
  50 void cpu_gen_init(void);
  51
  52 /**
  53  * cpu_restore_state:
  54  * @cpu: the vCPU state is to be restore to
  55  * @searched_pc: the host PC the fault occurred at
  56  * @will_exit: true if the TB executed will be interrupted after some
  57                cpu adjustments. Required for maintaining the correct
  58                icount valus
  59  * @return: true if state was restored, false otherwise
  60  *
  61  * Attempt to restore the state for a fault occurring in translated
  62  * code. If the searched_pc is not in translated code no state is
  63  * restored and the function returns false.
  64  */
  65 bool cpu_restore_state(CPUState *cpu, uintptr_t searched_pc, bool will_exit);
  66
  67 void QEMU_NORETURN cpu_loop_exit_noexc(CPUState *cpu);
  68 void QEMU_NORETURN cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
  69 TranslationBlock *tb_gen_code(CPUState *cpu,
  70                               target_ulong pc, target_ulong cs_base,
  71                               uint32_t flags,
  72                               int cflags);
  73
  74 void QEMU_NORETURN cpu_loop_exit(CPUState *cpu);
  75 void QEMU_NORETURN cpu_loop_exit_restore(CPUState *cpu, uintptr_t pc);
  76 void QEMU_NORETURN cpu_loop_exit_atomic(CPUState *cpu, uintptr_t pc);
  77
  78 #if !defined(CONFIG_USER_ONLY)
  79 void cpu_reloading_memory_map(void);
  80 /**
  81  * cpu_address_space_init:
  82  * @cpu: CPU to add this address space to
  83  * @asidx: integer index of this address space
  84  * @prefix: prefix to be used as name of address space
  85  * @mr: the root memory region of address space
  86  *
  87  * Add the specified address space to the CPU's cpu_ases list.
  88  * The address space added with @asidx 0 is the one used for the
  89  * convenience pointer cpu->as.
  90  * The target-specific code which registers ASes is responsible
  91  * for defining what semantics address space 0, 1, 2, etc have.
  92  *
  93  * Before the first call to this function, the caller must set
  94  * cpu->num_ases to the total number of address spaces it needs
  95  * to support.
  96  *
  97  * Note that with KVM only one address space is supported.
  98  */
  99 void cpu_address_space_init(CPUState *cpu, int asidx,
 100                             const char *prefix, MemoryRegion *mr);
 101 #endif
 102
 103 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_TCG)
 104 /* cputlb.c */
 105 /**
 106  * tlb_init - initialize a CPU's TLB
 107  * @cpu: CPU whose TLB should be initialized
 108  */
 109 void tlb_init(CPUState *cpu);
 110 /**
 111  * tlb_flush_page:
 112  * @cpu: CPU whose TLB should be flushed
 113  * @addr: virtual address of page to be flushed
 114  *
 115  * Flush one page from the TLB of the specified CPU, for all
 116  * MMU indexes.
 117  */
 118 void tlb_flush_page(CPUState *cpu, target_ulong addr);
 119 /**
 120  * tlb_flush_page_all_cpus:
 121  * @cpu: src CPU of the flush
 122  * @addr: virtual address of page to be flushed
 123  *
 124  * Flush one page from the TLB of the specified CPU, for all
 125  * MMU indexes.
 126  */
 127 void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr);
 128 /**
 129  * tlb_flush_page_all_cpus_synced:
 130  * @cpu: src CPU of the flush
 131  * @addr: virtual address of page to be flushed
 132  *
 133  * Flush one page from the TLB of the specified CPU, for all MMU
 134  * indexes like tlb_flush_page_all_cpus except the source vCPUs work
 135  * is scheduled as safe work meaning all flushes will be complete once
 136  * the source vCPUs safe work is complete. This will depend on when
 137  * the guests translation ends the TB.
 138  */
 139 void tlb_flush_page_all_cpus_synced(CPUState *src, target_ulong addr);
 140 /**
 141  * tlb_flush:
 142  * @cpu: CPU whose TLB should be flushed
 143  *
 144  * Flush the entire TLB for the specified CPU. Most CPU architectures
 145  * allow the implementation to drop entries from the TLB at any time
 146  * so this is generally safe. If more selective flushing is required
 147  * use one of the other functions for efficiency.
 148  */
 149 void tlb_flush(CPUState *cpu);
 150 /**
 151  * tlb_flush_all_cpus:
 152  * @cpu: src CPU of the flush
 153  */
 154 void tlb_flush_all_cpus(CPUState *src_cpu);
 155 /**
 156  * tlb_flush_all_cpus_synced:
 157  * @cpu: src CPU of the flush
 158  *
 159  * Like tlb_flush_all_cpus except this except the source vCPUs work is
 160  * scheduled as safe work meaning all flushes will be complete once
 161  * the source vCPUs safe work is complete. This will depend on when
 162  * the guests translation ends the TB.
 163  */
 164 void tlb_flush_all_cpus_synced(CPUState *src_cpu);
 165 /**
 166  * tlb_flush_page_by_mmuidx:
 167  * @cpu: CPU whose TLB should be flushed
 168  * @addr: virtual address of page to be flushed
 169  * @idxmap: bitmap of MMU indexes to flush
 170  *
 171  * Flush one page from the TLB of the specified CPU, for the specified
 172  * MMU indexes.
 173  */
 174 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr,
 175                               uint16_t idxmap);
 176 /**
 177  * tlb_flush_page_by_mmuidx_all_cpus:
 178  * @cpu: Originating CPU of the flush
 179  * @addr: virtual address of page to be flushed
 180  * @idxmap: bitmap of MMU indexes to flush
 181  *
 182  * Flush one page from the TLB of all CPUs, for the specified
 183  * MMU indexes.
 184  */
 185 void tlb_flush_page_by_mmuidx_all_cpus(CPUState *cpu, target_ulong addr,
 186                                        uint16_t idxmap);
 187 /**
 188  * tlb_flush_page_by_mmuidx_all_cpus_synced:
 189  * @cpu: Originating CPU of the flush
 190  * @addr: virtual address of page to be flushed
 191  * @idxmap: bitmap of MMU indexes to flush
 192  *
 193  * Flush one page from the TLB of all CPUs, for the specified MMU
 194  * indexes like tlb_flush_page_by_mmuidx_all_cpus except the source
 195  * vCPUs work is scheduled as safe work meaning all flushes will be
 196  * complete once  the source vCPUs safe work is complete. This will
 197  * depend on when the guests translation ends the TB.
 198  */
 199 void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu, target_ulong addr,
 200                                               uint16_t idxmap);
 201 /**
 202  * tlb_flush_by_mmuidx:
 203  * @cpu: CPU whose TLB should be flushed
 204  * @wait: If true ensure synchronisation by exiting the cpu_loop
 205  * @idxmap: bitmap of MMU indexes to flush
 206  *
 207  * Flush all entries from the TLB of the specified CPU, for the specified
 208  * MMU indexes.
 209  */
 210 void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap);
 211 /**
 212  * tlb_flush_by_mmuidx_all_cpus:
 213  * @cpu: Originating CPU of the flush
 214  * @idxmap: bitmap of MMU indexes to flush
 215  *
 216  * Flush all entries from all TLBs of all CPUs, for the specified
 217  * MMU indexes.
 218  */
 219 void tlb_flush_by_mmuidx_all_cpus(CPUState *cpu, uint16_t idxmap);
 220 /**
 221  * tlb_flush_by_mmuidx_all_cpus_synced:
 222  * @cpu: Originating CPU of the flush
 223  * @idxmap: bitmap of MMU indexes to flush
 224  *
 225  * Flush all entries from all TLBs of all CPUs, for the specified
 226  * MMU indexes like tlb_flush_by_mmuidx_all_cpus except except the source
 227  * vCPUs work is scheduled as safe work meaning all flushes will be
 228  * complete once  the source vCPUs safe work is complete. This will
 229  * depend on when the guests translation ends the TB.
 230  */
 231 void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu, uint16_t idxmap);
 232 /**
 233  * tlb_set_page_with_attrs:
 234  * @cpu: CPU to add this TLB entry for
 235  * @vaddr: virtual address of page to add entry for
 236  * @paddr: physical address of the page
 237  * @attrs: memory transaction attributes
 238  * @prot: access permissions (PAGE_READ/PAGE_WRITE/PAGE_EXEC bits)
 239  * @mmu_idx: MMU index to insert TLB entry for
 240  * @size: size of the page in bytes
 241  *
 242  * Add an entry to this CPU's TLB (a mapping from virtual address
 243  * @vaddr to physical address @paddr) with the specified memory
 244  * transaction attributes. This is generally called by the target CPU
 245  * specific code after it has been called through the tlb_fill()
 246  * entry point and performed a successful page table walk to find
 247  * the physical address and attributes for the virtual address
 248  * which provoked the TLB miss.
 249  *
 250  * At most one entry for a given virtual address is permitted. Only a
 251  * single TARGET_PAGE_SIZE region is mapped; the supplied @size is only
 252  * used by tlb_flush_page.
 253  */
 254 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
 255                              hwaddr paddr, MemTxAttrs attrs,
 256                              int prot, int mmu_idx, target_ulong size);
 257 /* tlb_set_page:
 258  *
 259  * This function is equivalent to calling tlb_set_page_with_attrs()
 260  * with an @attrs argument of MEMTXATTRS_UNSPECIFIED. It's provided
 261  * as a convenience for CPUs which don't use memory transaction attributes.
 262  */
 263 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
 264                   hwaddr paddr, int prot,
 265                   int mmu_idx, target_ulong size);
 266 void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
 267                  uintptr_t retaddr);
 268 #else
 269 static inline void tlb_init(CPUState *cpu)
 270 {
 271 }
 272 static inline void tlb_flush_page(CPUState *cpu, target_ulong addr)
 273 {
 274 }
 275 static inline void tlb_flush_page_all_cpus(CPUState *src, target_ulong addr)
 276 {
 277 }
 278 static inline void tlb_flush_page_all_cpus_synced(CPUState *src,
 279                                                   target_ulong addr)
 280 {
 281 }
 282 static inline void tlb_flush(CPUState *cpu)
 283 {
 284 }
 285 static inline void tlb_flush_all_cpus(CPUState *src_cpu)
 286 {
 287 }
 288 static inline void tlb_flush_all_cpus_synced(CPUState *src_cpu)
 289 {
 290 }
 291 static inline void tlb_flush_page_by_mmuidx(CPUState *cpu,
 292                                             target_ulong addr, uint16_t idxmap)
 293 {
 294 }
 295
 296 static inline void tlb_flush_by_mmuidx(CPUState *cpu, uint16_t idxmap)
 297 {
 298 }
 299 static inline void tlb_flush_page_by_mmuidx_all_cpus(CPUState *cpu,
 300                                                      target_ulong addr,
 301                                                      uint16_t idxmap)
 302 {
 303 }
 304 static inline void tlb_flush_page_by_mmuidx_all_cpus_synced(CPUState *cpu,
 305                                                             target_ulong addr,
 306                                                             uint16_t idxmap)
 307 {
 308 }
 309 static inline void tlb_flush_by_mmuidx_all_cpus(CPUState *cpu, uint16_t idxmap)
 310 {
 311 }
 312
 313 static inline void tlb_flush_by_mmuidx_all_cpus_synced(CPUState *cpu,
 314                                                        uint16_t idxmap)
 315 {
 316 }
 317 #endif
 318
 319 #define CODE_GEN_ALIGN           16 /* must be >= of the size of a icache line */
 320
 321 /* Estimated block size for TB allocation.  */
 322 /* ??? The following is based on a 2015 survey of x86_64 host output.
 323    Better would seem to be some sort of dynamically sized TB array,
 324    adapting to the block sizes actually being produced.  */
 325 #if defined(CONFIG_SOFTMMU)
 326 #define CODE_GEN_AVG_BLOCK_SIZE 400
 327 #else
 328 #define CODE_GEN_AVG_BLOCK_SIZE 150
 329 #endif
 330
 331 /*
 332  * Translation Cache-related fields of a TB.
 333  * This struct exists just for convenience; we keep track of TB's in a binary
 334  * search tree, and the only fields needed to compare TB's in the tree are
 335  * @ptr and @size.
 336  * Note: the address of search data can be obtained by adding @size to @ptr.
 337  */
 338 struct tb_tc {
 339     void *ptr;    /* pointer to the translated code */
 340     size_t size;
 341 };
 342
 343 struct TranslationBlock {
 344     target_ulong pc;   /* simulated PC corresponding to this block (EIP + CS base) */
 345     target_ulong cs_base; /* CS base for this block */
 346     uint32_t flags; /* flags defining in which context the code was generated */
 347     uint16_t size;      /* size of target code for this block (1 <=
 348                            size <= TARGET_PAGE_SIZE) */
 349     uint16_t icount;
 350     uint32_t cflags;    /* compile flags */
 351 #define CF_COUNT_MASK  0x00007fff
 352 #define CF_LAST_IO     0x00008000 /* Last insn may be an IO access.  */
 353 #define CF_NOCACHE     0x00010000 /* To be freed after execution */
 354 #define CF_USE_ICOUNT  0x00020000
 355 #define CF_INVALID     0x00040000 /* TB is stale. Set with @jmp_lock held */
 356 #define CF_PARALLEL    0x00080000 /* Generate code for a parallel context */
 357 /* cflags' mask for hashing/comparison */
 358 #define CF_HASH_MASK   \
 359     (CF_COUNT_MASK | CF_LAST_IO | CF_USE_ICOUNT | CF_PARALLEL)
 360
 361     /* Per-vCPU dynamic tracing state used to generate this TB */
 362     uint32_t trace_vcpu_dstate;
 363
 364     struct tb_tc tc;
 365
 366     /* original tb when cflags has CF_NOCACHE */
 367     struct TranslationBlock *orig_tb;
 368     /* first and second physical page containing code. The lower bit
 369        of the pointer tells the index in page_next[].
 370        The list is protected by the TB's page('s) lock(s) */
 371     uintptr_t page_next[2];
 372     tb_page_addr_t page_addr[2];
 373
 374     /* jmp_lock placed here to fill a 4-byte hole. Its documentation is below */
 375     QemuSpin jmp_lock;
 376
 377     /* The following data are used to directly call another TB from
 378      * the code of this one. This can be done either by emitting direct or
 379      * indirect native jump instructions. These jumps are reset so that the TB
 380      * just continues its execution. The TB can be linked to another one by
 381      * setting one of the jump targets (or patching the jump instruction). Only
 382      * two of such jumps are supported.
 383      */
 384     uint16_t jmp_reset_offset[2]; /* offset of original jump target */
 385 #define TB_JMP_RESET_OFFSET_INVALID 0xffff /* indicates no jump generated */
 386     uintptr_t jmp_target_arg[2];  /* target address or offset */
 387
 388     /*
 389      * Each TB has a NULL-terminated list (jmp_list_head) of incoming jumps.
 390      * Each TB can have two outgoing jumps, and therefore can participate
 391      * in two lists. The list entries are kept in jmp_list_next[2]. The least
 392      * significant bit (LSB) of the pointers in these lists is used to encode
 393      * which of the two list entries is to be used in the pointed TB.
 394      *
 395      * List traversals are protected by jmp_lock. The destination TB of each
 396      * outgoing jump is kept in jmp_dest[] so that the appropriate jmp_lock
 397      * can be acquired from any origin TB.
 398      *
 399      * jmp_dest[] are tagged pointers as well. The LSB is set when the TB is
 400      * being invalidated, so that no further outgoing jumps from it can be set.
 401      *
 402      * jmp_lock also protects the CF_INVALID cflag; a jump must not be chained
 403      * to a destination TB that has CF_INVALID set.
 404      */
 405     uintptr_t jmp_list_head;
 406     uintptr_t jmp_list_next[2];
 407     uintptr_t jmp_dest[2];
 408 };
 409
 410 extern bool parallel_cpus;
 411
 412 /* Hide the atomic_read to make code a little easier on the eyes */
 413 static inline uint32_t tb_cflags(const TranslationBlock *tb)
 414 {
 415     return atomic_read(&tb->cflags);
 416 }
 417
 418 /* current cflags for hashing/comparison */
 419 static inline uint32_t curr_cflags(void)
 420 {
 421     return (parallel_cpus ? CF_PARALLEL : 0)
 422          | (use_icount ? CF_USE_ICOUNT : 0);
 423 }
 424
 425 /* TranslationBlock invalidate API */
 426 #if defined(CONFIG_USER_ONLY)
 427 void tb_invalidate_phys_addr(target_ulong addr);
 428 void tb_invalidate_phys_range(target_ulong start, target_ulong end);
 429 #else
 430 void tb_invalidate_phys_addr(AddressSpace *as, hwaddr addr, MemTxAttrs attrs);
 431 #endif
 432 void tb_flush(CPUState *cpu);
 433 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
 434 TranslationBlock *tb_htable_lookup(CPUState *cpu, target_ulong pc,
 435                                    target_ulong cs_base, uint32_t flags,
 436                                    uint32_t cf_mask);
 437 void tb_set_jmp_target(TranslationBlock *tb, int n, uintptr_t addr);
 438
 439 /* GETPC is the true target of the return instruction that we'll execute.  */
 440 #if defined(CONFIG_TCG_INTERPRETER)
 441 extern uintptr_t tci_tb_ptr;
 442 # define GETPC() tci_tb_ptr
 443 #else
 444 # define GETPC() \
 445     ((uintptr_t)__builtin_extract_return_addr(__builtin_return_address(0)))
 446 #endif
 447
 448 /* The true return address will often point to a host insn that is part of
 449    the next translated guest insn.  Adjust the address backward to point to
 450    the middle of the call insn.  Subtracting one would do the job except for
 451    several compressed mode architectures (arm, mips) which set the low bit
 452    to indicate the compressed mode; subtracting two works around that.  It
 453    is also the case that there are no host isas that contain a call insn
 454    smaller than 4 bytes, so we don't worry about special-casing this.  */
 455 #define GETPC_ADJ   2
 456
 457 #if !defined(CONFIG_USER_ONLY) && defined(CONFIG_DEBUG_TCG)
 458 void assert_no_pages_locked(void);
 459 #else
 460 static inline void assert_no_pages_locked(void)
 461 {
 462 }
 463 #endif
 464
 465 #if !defined(CONFIG_USER_ONLY)
 466
 467 /**
 468  * iotlb_to_section:
 469  * @cpu: CPU performing the access
 470  * @index: TCG CPU IOTLB entry
 471  *
 472  * Given a TCG CPU IOTLB entry, return the MemoryRegionSection that
 473  * it refers to. @index will have been initially created and returned
 474  * by memory_region_section_get_iotlb().
 475  */
 476 struct MemoryRegionSection *iotlb_to_section(CPUState *cpu,
 477                                              hwaddr index, MemTxAttrs attrs);
 478
 479 void tlb_fill(CPUState *cpu, target_ulong addr, int size,
 480               MMUAccessType access_type, int mmu_idx, uintptr_t retaddr);
 481
 482 #endif
 483
 484 #if defined(CONFIG_USER_ONLY)
 485 void mmap_lock(void);
 486 void mmap_unlock(void);
 487 bool have_mmap_lock(void);
 488
 489 static inline tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 490 {
 491     return addr;
 492 }
 493 #else
 494 static inline void mmap_lock(void) {}
 495 static inline void mmap_unlock(void) {}
 496
 497 /* cputlb.c */
 498 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr);
 499
 500 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length);
 501 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr);
 502
 503 /* exec.c */
 504 void tb_flush_jmp_cache(CPUState *cpu, target_ulong addr);
 505
 506 MemoryRegionSection *
 507 address_space_translate_for_iotlb(CPUState *cpu, int asidx, hwaddr addr,
 508                                   hwaddr *xlat, hwaddr *plen,
 509                                   MemTxAttrs attrs, int *prot);
 510 hwaddr memory_region_section_get_iotlb(CPUState *cpu,
 511                                        MemoryRegionSection *section,
 512                                        target_ulong vaddr,
 513                                        hwaddr paddr, hwaddr xlat,
 514                                        int prot,
 515                                        target_ulong *address);
 516 #endif
 517
 518 /* vl.c */
 519 extern int singlestep;
 520
 521 #endif