cputlb.c

   1 /*
   2  *  Common CPU TLB handling
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "cpu.h"
  22 #include "exec/exec-all.h"
  23 #include "exec/memory.h"
  24 #include "exec/address-spaces.h"
  25 #include "exec/cpu_ldst.h"
  26
  27 #include "exec/cputlb.h"
  28
  29 #include "exec/memory-internal.h"
  30 #include "exec/ram_addr.h"
  31 #include "exec/exec-all.h"
  32 #include "tcg/tcg.h"
  33
  34 /* DEBUG defines, enable DEBUG_TLB_LOG to log to the CPU_LOG_MMU target */
  35 /* #define DEBUG_TLB */
  36 /* #define DEBUG_TLB_LOG */
  37
  38 #ifdef DEBUG_TLB
  39 # define DEBUG_TLB_GATE 1
  40 # ifdef DEBUG_TLB_LOG
  41 #  define DEBUG_TLB_LOG_GATE 1
  42 # else
  43 #  define DEBUG_TLB_LOG_GATE 0
  44 # endif
  45 #else
  46 # define DEBUG_TLB_GATE 0
  47 # define DEBUG_TLB_LOG_GATE 0
  48 #endif
  49
  50 #define tlb_debug(fmt, ...) do { \
  51     if (DEBUG_TLB_LOG_GATE) { \
  52         qemu_log_mask(CPU_LOG_MMU, "%s: " fmt, __func__, \
  53                       ## __VA_ARGS__); \
  54     } else if (DEBUG_TLB_GATE) { \
  55         fprintf(stderr, "%s: " fmt, __func__, ## __VA_ARGS__); \
  56     } \
  57 } while (0)
  58
  59 /* statistics */
  60 int tlb_flush_count;
  61
  62 /* NOTE:
  63  * If flush_global is true (the usual case), flush all tlb entries.
  64  * If flush_global is false, flush (at least) all tlb entries not
  65  * marked global.
  66  *
  67  * Since QEMU doesn't currently implement a global/not-global flag
  68  * for tlb entries, at the moment tlb_flush() will also flush all
  69  * tlb entries in the flush_global == false case. This is OK because
  70  * CPU architectures generally permit an implementation to drop
  71  * entries from the TLB at any time, so flushing more entries than
  72  * required is only an efficiency issue, not a correctness issue.
  73  */
  74 void tlb_flush(CPUState *cpu, int flush_global)
  75 {
  76     CPUArchState *env = cpu->env_ptr;
  77
  78     tlb_debug("(%d)\n", flush_global);
  79
  80     memset(env->tlb_table, -1, sizeof(env->tlb_table));
  81     memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
  82     memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
  83
  84     env->vtlb_index = 0;
  85     env->tlb_flush_addr = -1;
  86     env->tlb_flush_mask = 0;
  87     tlb_flush_count++;
  88 }
  89
  90 static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
  91 {
  92     CPUArchState *env = cpu->env_ptr;
  93
  94     tlb_debug("start\n");
  95
  96     for (;;) {
  97         int mmu_idx = va_arg(argp, int);
  98
  99         if (mmu_idx < 0) {
 100             break;
 101         }
 102
 103         tlb_debug("%d\n", mmu_idx);
 104
 105         memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
 106         memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
 107     }
 108
 109     memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
 110 }
 111
 112 void tlb_flush_by_mmuidx(CPUState *cpu, ...)
 113 {
 114     va_list argp;
 115     va_start(argp, cpu);
 116     v_tlb_flush_by_mmuidx(cpu, argp);
 117     va_end(argp);
 118 }
 119
 120 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
 121 {
 122     if (addr == (tlb_entry->addr_read &
 123                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
 124         addr == (tlb_entry->addr_write &
 125                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
 126         addr == (tlb_entry->addr_code &
 127                  (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 128         memset(tlb_entry, -1, sizeof(*tlb_entry));
 129     }
 130 }
 131
 132 void tlb_flush_page(CPUState *cpu, target_ulong addr)
 133 {
 134     CPUArchState *env = cpu->env_ptr;
 135     int i;
 136     int mmu_idx;
 137
 138     tlb_debug("page :" TARGET_FMT_lx "\n", addr);
 139
 140     /* Check if we need to flush due to large pages.  */
 141     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
 142         tlb_debug("forcing full flush ("
 143                   TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
 144                   env->tlb_flush_addr, env->tlb_flush_mask);
 145
 146         tlb_flush(cpu, 1);
 147         return;
 148     }
 149
 150     addr &= TARGET_PAGE_MASK;
 151     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 152     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 153         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
 154     }
 155
 156     /* check whether there are entries that need to be flushed in the vtlb */
 157     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 158         int k;
 159         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 160             tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
 161         }
 162     }
 163
 164     tb_flush_jmp_cache(cpu, addr);
 165 }
 166
 167 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
 168 {
 169     CPUArchState *env = cpu->env_ptr;
 170     int i, k;
 171     va_list argp;
 172
 173     va_start(argp, addr);
 174
 175     tlb_debug("addr "TARGET_FMT_lx"\n", addr);
 176
 177     /* Check if we need to flush due to large pages.  */
 178     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
 179         tlb_debug("forced full flush ("
 180                   TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
 181                   env->tlb_flush_addr, env->tlb_flush_mask);
 182
 183         v_tlb_flush_by_mmuidx(cpu, argp);
 184         va_end(argp);
 185         return;
 186     }
 187
 188     addr &= TARGET_PAGE_MASK;
 189     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 190
 191     for (;;) {
 192         int mmu_idx = va_arg(argp, int);
 193
 194         if (mmu_idx < 0) {
 195             break;
 196         }
 197
 198         tlb_debug("idx %d\n", mmu_idx);
 199
 200         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
 201
 202         /* check whether there are vltb entries that need to be flushed */
 203         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 204             tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
 205         }
 206     }
 207     va_end(argp);
 208
 209     tb_flush_jmp_cache(cpu, addr);
 210 }
 211
 212 /* update the TLBs so that writes to code in the virtual page 'addr'
 213    can be detected */
 214 void tlb_protect_code(ram_addr_t ram_addr)
 215 {
 216     cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
 217                                              DIRTY_MEMORY_CODE);
 218 }
 219
 220 /* update the TLB so that writes in physical page 'phys_addr' are no longer
 221    tested for self modifying code */
 222 void tlb_unprotect_code(ram_addr_t ram_addr)
 223 {
 224     cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
 225 }
 226
 227 static bool tlb_is_dirty_ram(CPUTLBEntry *tlbe)
 228 {
 229     return (tlbe->addr_write & (TLB_INVALID_MASK|TLB_MMIO|TLB_NOTDIRTY)) == 0;
 230 }
 231
 232 void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
 233                            uintptr_t length)
 234 {
 235     uintptr_t addr;
 236
 237     if (tlb_is_dirty_ram(tlb_entry)) {
 238         addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
 239         if ((addr - start) < length) {
 240             tlb_entry->addr_write |= TLB_NOTDIRTY;
 241         }
 242     }
 243 }
 244
 245 static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
 246 {
 247     ram_addr_t ram_addr;
 248
 249     if (qemu_ram_addr_from_host(ptr, &ram_addr) == NULL) {
 250         fprintf(stderr, "Bad ram pointer %p\n", ptr);
 251         abort();
 252     }
 253     return ram_addr;
 254 }
 255
 256 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 257 {
 258     CPUArchState *env;
 259
 260     int mmu_idx;
 261
 262     env = cpu->env_ptr;
 263     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 264         unsigned int i;
 265
 266         for (i = 0; i < CPU_TLB_SIZE; i++) {
 267             tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
 268                                   start1, length);
 269         }
 270
 271         for (i = 0; i < CPU_VTLB_SIZE; i++) {
 272             tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
 273                                   start1, length);
 274         }
 275     }
 276 }
 277
 278 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
 279 {
 280     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
 281         tlb_entry->addr_write = vaddr;
 282     }
 283 }
 284
 285 /* update the TLB corresponding to virtual page vaddr
 286    so that it is no longer dirty */
 287 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 288 {
 289     CPUArchState *env = cpu->env_ptr;
 290     int i;
 291     int mmu_idx;
 292
 293     vaddr &= TARGET_PAGE_MASK;
 294     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 295     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 296         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
 297     }
 298
 299     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 300         int k;
 301         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 302             tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
 303         }
 304     }
 305 }
 306
 307 /* Our TLB does not support large pages, so remember the area covered by
 308    large pages and trigger a full TLB flush if these are invalidated.  */
 309 static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
 310                                target_ulong size)
 311 {
 312     target_ulong mask = ~(size - 1);
 313
 314     if (env->tlb_flush_addr == (target_ulong)-1) {
 315         env->tlb_flush_addr = vaddr & mask;
 316         env->tlb_flush_mask = mask;
 317         return;
 318     }
 319     /* Extend the existing region to include the new page.
 320        This is a compromise between unnecessary flushes and the cost
 321        of maintaining a full variable size TLB.  */
 322     mask &= env->tlb_flush_mask;
 323     while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
 324         mask <<= 1;
 325     }
 326     env->tlb_flush_addr &= mask;
 327     env->tlb_flush_mask = mask;
 328 }
 329
 330 /* Add a new TLB entry. At most one entry for a given virtual address
 331  * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
 332  * supplied size is only used by tlb_flush_page.
 333  *
 334  * Called from TCG-generated code, which is under an RCU read-side
 335  * critical section.
 336  */
 337 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
 338                              hwaddr paddr, MemTxAttrs attrs, int prot,
 339                              int mmu_idx, target_ulong size)
 340 {
 341     CPUArchState *env = cpu->env_ptr;
 342     MemoryRegionSection *section;
 343     unsigned int index;
 344     target_ulong address;
 345     target_ulong code_address;
 346     uintptr_t addend;
 347     CPUTLBEntry *te;
 348     hwaddr iotlb, xlat, sz;
 349     unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
 350     int asidx = cpu_asidx_from_attrs(cpu, attrs);
 351
 352     assert(size >= TARGET_PAGE_SIZE);
 353     if (size != TARGET_PAGE_SIZE) {
 354         tlb_add_large_page(env, vaddr, size);
 355     }
 356
 357     sz = size;
 358     section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
 359     assert(sz >= TARGET_PAGE_SIZE);
 360
 361     tlb_debug("vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
 362               " prot=%x idx=%d\n",
 363               vaddr, paddr, prot, mmu_idx);
 364
 365     address = vaddr;
 366     if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
 367         /* IO memory case */
 368         address |= TLB_MMIO;
 369         addend = 0;
 370     } else {
 371         /* TLB_MMIO for rom/romd handled below */
 372         addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
 373     }
 374
 375     code_address = address;
 376     iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
 377                                             prot, &address);
 378
 379     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 380     te = &env->tlb_table[mmu_idx][index];
 381
 382     /* do not discard the translation in te, evict it into a victim tlb */
 383     env->tlb_v_table[mmu_idx][vidx] = *te;
 384     env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
 385
 386     /* refill the tlb */
 387     env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
 388     env->iotlb[mmu_idx][index].attrs = attrs;
 389     te->addend = addend - vaddr;
 390     if (prot & PAGE_READ) {
 391         te->addr_read = address;
 392     } else {
 393         te->addr_read = -1;
 394     }
 395
 396     if (prot & PAGE_EXEC) {
 397         te->addr_code = code_address;
 398     } else {
 399         te->addr_code = -1;
 400     }
 401     if (prot & PAGE_WRITE) {
 402         if ((memory_region_is_ram(section->mr) && section->readonly)
 403             || memory_region_is_romd(section->mr)) {
 404             /* Write access calls the I/O callback.  */
 405             te->addr_write = address | TLB_MMIO;
 406         } else if (memory_region_is_ram(section->mr)
 407                    && cpu_physical_memory_is_clean(
 408                         memory_region_get_ram_addr(section->mr) + xlat)) {
 409             te->addr_write = address | TLB_NOTDIRTY;
 410         } else {
 411             te->addr_write = address;
 412         }
 413     } else {
 414         te->addr_write = -1;
 415     }
 416 }
 417
 418 /* Add a new TLB entry, but without specifying the memory
 419  * transaction attributes to be used.
 420  */
 421 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
 422                   hwaddr paddr, int prot,
 423                   int mmu_idx, target_ulong size)
 424 {
 425     tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
 426                             prot, mmu_idx, size);
 427 }
 428
 429 /* NOTE: this function can trigger an exception */
 430 /* NOTE2: the returned address is not exactly the physical address: it
 431  * is actually a ram_addr_t (in system mode; the user mode emulation
 432  * version of this function returns a guest virtual address).
 433  */
 434 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 435 {
 436     int mmu_idx, page_index, pd;
 437     void *p;
 438     MemoryRegion *mr;
 439     CPUState *cpu = ENV_GET_CPU(env1);
 440     CPUIOTLBEntry *iotlbentry;
 441
 442     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 443     mmu_idx = cpu_mmu_index(env1, true);
 444     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
 445                  (addr & TARGET_PAGE_MASK))) {
 446         cpu_ldub_code(env1, addr);
 447     }
 448     iotlbentry = &env1->iotlb[mmu_idx][page_index];
 449     pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
 450     mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
 451     if (memory_region_is_unassigned(mr)) {
 452         CPUClass *cc = CPU_GET_CLASS(cpu);
 453
 454         if (cc->do_unassigned_access) {
 455             cc->do_unassigned_access(cpu, addr, false, true, 0, 4);
 456         } else {
 457             cpu_abort(cpu, "Trying to execute code outside RAM or ROM at 0x"
 458                       TARGET_FMT_lx "\n", addr);
 459         }
 460     }
 461     p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
 462     return qemu_ram_addr_from_host_nofail(p);
 463 }
 464
 465 #define MMUSUFFIX _mmu
 466
 467 #define SHIFT 0
 468 #include "softmmu_template.h"
 469
 470 #define SHIFT 1
 471 #include "softmmu_template.h"
 472
 473 #define SHIFT 2
 474 #include "softmmu_template.h"
 475
 476 #define SHIFT 3
 477 #include "softmmu_template.h"
 478 #undef MMUSUFFIX
 479
 480 #define MMUSUFFIX _cmmu
 481 #undef GETPC_ADJ
 482 #define GETPC_ADJ 0
 483 #undef GETRA
 484 #define GETRA() ((uintptr_t)0)
 485 #define SOFTMMU_CODE_ACCESS
 486
 487 #define SHIFT 0
 488 #include "softmmu_template.h"
 489
 490 #define SHIFT 1
 491 #include "softmmu_template.h"
 492
 493 #define SHIFT 2
 494 #include "softmmu_template.h"
 495
 496 #define SHIFT 3
 497 #include "softmmu_template.h"