cputlb.c

   1 /*
   2  *  Common CPU TLB handling
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "cpu.h"
  22 #include "exec/exec-all.h"
  23 #include "exec/memory.h"
  24 #include "exec/address-spaces.h"
  25 #include "exec/cpu_ldst.h"
  26
  27 #include "exec/cputlb.h"
  28
  29 #include "exec/memory-internal.h"
  30 #include "exec/ram_addr.h"
  31 #include "tcg/tcg.h"
  32
  33 //#define DEBUG_TLB
  34 //#define DEBUG_TLB_CHECK
  35
  36 /* statistics */
  37 int tlb_flush_count;
  38
  39 /* NOTE:
  40  * If flush_global is true (the usual case), flush all tlb entries.
  41  * If flush_global is false, flush (at least) all tlb entries not
  42  * marked global.
  43  *
  44  * Since QEMU doesn't currently implement a global/not-global flag
  45  * for tlb entries, at the moment tlb_flush() will also flush all
  46  * tlb entries in the flush_global == false case. This is OK because
  47  * CPU architectures generally permit an implementation to drop
  48  * entries from the TLB at any time, so flushing more entries than
  49  * required is only an efficiency issue, not a correctness issue.
  50  */
  51 void tlb_flush(CPUState *cpu, int flush_global)
  52 {
  53     CPUArchState *env = cpu->env_ptr;
  54
  55 #if defined(DEBUG_TLB)
  56     printf("tlb_flush:\n");
  57 #endif
  58     /* must reset current TB so that interrupts cannot modify the
  59        links while we are modifying them */
  60     cpu->current_tb = NULL;
  61
  62     memset(env->tlb_table, -1, sizeof(env->tlb_table));
  63     memset(env->tlb_v_table, -1, sizeof(env->tlb_v_table));
  64     memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
  65
  66     env->vtlb_index = 0;
  67     env->tlb_flush_addr = -1;
  68     env->tlb_flush_mask = 0;
  69     tlb_flush_count++;
  70 }
  71
  72 static inline void v_tlb_flush_by_mmuidx(CPUState *cpu, va_list argp)
  73 {
  74     CPUArchState *env = cpu->env_ptr;
  75
  76 #if defined(DEBUG_TLB)
  77     printf("tlb_flush_by_mmuidx:");
  78 #endif
  79     /* must reset current TB so that interrupts cannot modify the
  80        links while we are modifying them */
  81     cpu->current_tb = NULL;
  82
  83     for (;;) {
  84         int mmu_idx = va_arg(argp, int);
  85
  86         if (mmu_idx < 0) {
  87             break;
  88         }
  89
  90 #if defined(DEBUG_TLB)
  91         printf(" %d", mmu_idx);
  92 #endif
  93
  94         memset(env->tlb_table[mmu_idx], -1, sizeof(env->tlb_table[0]));
  95         memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
  96     }
  97
  98 #if defined(DEBUG_TLB)
  99     printf("\n");
 100 #endif
 101
 102     memset(cpu->tb_jmp_cache, 0, sizeof(cpu->tb_jmp_cache));
 103 }
 104
 105 void tlb_flush_by_mmuidx(CPUState *cpu, ...)
 106 {
 107     va_list argp;
 108     va_start(argp, cpu);
 109     v_tlb_flush_by_mmuidx(cpu, argp);
 110     va_end(argp);
 111 }
 112
 113 static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
 114 {
 115     if (addr == (tlb_entry->addr_read &
 116                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
 117         addr == (tlb_entry->addr_write &
 118                  (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
 119         addr == (tlb_entry->addr_code &
 120                  (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
 121         memset(tlb_entry, -1, sizeof(*tlb_entry));
 122     }
 123 }
 124
 125 void tlb_flush_page(CPUState *cpu, target_ulong addr)
 126 {
 127     CPUArchState *env = cpu->env_ptr;
 128     int i;
 129     int mmu_idx;
 130
 131 #if defined(DEBUG_TLB)
 132     printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
 133 #endif
 134     /* Check if we need to flush due to large pages.  */
 135     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
 136 #if defined(DEBUG_TLB)
 137         printf("tlb_flush_page: forced full flush ("
 138                TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
 139                env->tlb_flush_addr, env->tlb_flush_mask);
 140 #endif
 141         tlb_flush(cpu, 1);
 142         return;
 143     }
 144     /* must reset current TB so that interrupts cannot modify the
 145        links while we are modifying them */
 146     cpu->current_tb = NULL;
 147
 148     addr &= TARGET_PAGE_MASK;
 149     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 150     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 151         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
 152     }
 153
 154     /* check whether there are entries that need to be flushed in the vtlb */
 155     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 156         int k;
 157         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 158             tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
 159         }
 160     }
 161
 162     tb_flush_jmp_cache(cpu, addr);
 163 }
 164
 165 void tlb_flush_page_by_mmuidx(CPUState *cpu, target_ulong addr, ...)
 166 {
 167     CPUArchState *env = cpu->env_ptr;
 168     int i, k;
 169     va_list argp;
 170
 171     va_start(argp, addr);
 172
 173 #if defined(DEBUG_TLB)
 174     printf("tlb_flush_page_by_mmu_idx: " TARGET_FMT_lx, addr);
 175 #endif
 176     /* Check if we need to flush due to large pages.  */
 177     if ((addr & env->tlb_flush_mask) == env->tlb_flush_addr) {
 178 #if defined(DEBUG_TLB)
 179         printf(" forced full flush ("
 180                TARGET_FMT_lx "/" TARGET_FMT_lx ")\n",
 181                env->tlb_flush_addr, env->tlb_flush_mask);
 182 #endif
 183         v_tlb_flush_by_mmuidx(cpu, argp);
 184         va_end(argp);
 185         return;
 186     }
 187     /* must reset current TB so that interrupts cannot modify the
 188        links while we are modifying them */
 189     cpu->current_tb = NULL;
 190
 191     addr &= TARGET_PAGE_MASK;
 192     i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 193
 194     for (;;) {
 195         int mmu_idx = va_arg(argp, int);
 196
 197         if (mmu_idx < 0) {
 198             break;
 199         }
 200
 201 #if defined(DEBUG_TLB)
 202         printf(" %d", mmu_idx);
 203 #endif
 204
 205         tlb_flush_entry(&env->tlb_table[mmu_idx][i], addr);
 206
 207         /* check whether there are vltb entries that need to be flushed */
 208         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 209             tlb_flush_entry(&env->tlb_v_table[mmu_idx][k], addr);
 210         }
 211     }
 212     va_end(argp);
 213
 214 #if defined(DEBUG_TLB)
 215     printf("\n");
 216 #endif
 217
 218     tb_flush_jmp_cache(cpu, addr);
 219 }
 220
 221 /* update the TLBs so that writes to code in the virtual page 'addr'
 222    can be detected */
 223 void tlb_protect_code(ram_addr_t ram_addr)
 224 {
 225     cpu_physical_memory_test_and_clear_dirty(ram_addr, TARGET_PAGE_SIZE,
 226                                              DIRTY_MEMORY_CODE);
 227 }
 228
 229 /* update the TLB so that writes in physical page 'phys_addr' are no longer
 230    tested for self modifying code */
 231 void tlb_unprotect_code(ram_addr_t ram_addr)
 232 {
 233     cpu_physical_memory_set_dirty_flag(ram_addr, DIRTY_MEMORY_CODE);
 234 }
 235
 236 static bool tlb_is_dirty_ram(CPUTLBEntry *tlbe)
 237 {
 238     return (tlbe->addr_write & (TLB_INVALID_MASK|TLB_MMIO|TLB_NOTDIRTY)) == 0;
 239 }
 240
 241 void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
 242                            uintptr_t length)
 243 {
 244     uintptr_t addr;
 245
 246     if (tlb_is_dirty_ram(tlb_entry)) {
 247         addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
 248         if ((addr - start) < length) {
 249             tlb_entry->addr_write |= TLB_NOTDIRTY;
 250         }
 251     }
 252 }
 253
 254 static inline ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
 255 {
 256     ram_addr_t ram_addr;
 257
 258     if (qemu_ram_addr_from_host(ptr, &ram_addr) == NULL) {
 259         fprintf(stderr, "Bad ram pointer %p\n", ptr);
 260         abort();
 261     }
 262     return ram_addr;
 263 }
 264
 265 void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
 266 {
 267     CPUArchState *env;
 268
 269     int mmu_idx;
 270
 271     env = cpu->env_ptr;
 272     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 273         unsigned int i;
 274
 275         for (i = 0; i < CPU_TLB_SIZE; i++) {
 276             tlb_reset_dirty_range(&env->tlb_table[mmu_idx][i],
 277                                   start1, length);
 278         }
 279
 280         for (i = 0; i < CPU_VTLB_SIZE; i++) {
 281             tlb_reset_dirty_range(&env->tlb_v_table[mmu_idx][i],
 282                                   start1, length);
 283         }
 284     }
 285 }
 286
 287 static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
 288 {
 289     if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY)) {
 290         tlb_entry->addr_write = vaddr;
 291     }
 292 }
 293
 294 /* update the TLB corresponding to virtual page vaddr
 295    so that it is no longer dirty */
 296 void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
 297 {
 298     CPUArchState *env = cpu->env_ptr;
 299     int i;
 300     int mmu_idx;
 301
 302     vaddr &= TARGET_PAGE_MASK;
 303     i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 304     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 305         tlb_set_dirty1(&env->tlb_table[mmu_idx][i], vaddr);
 306     }
 307
 308     for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
 309         int k;
 310         for (k = 0; k < CPU_VTLB_SIZE; k++) {
 311             tlb_set_dirty1(&env->tlb_v_table[mmu_idx][k], vaddr);
 312         }
 313     }
 314 }
 315
 316 /* Our TLB does not support large pages, so remember the area covered by
 317    large pages and trigger a full TLB flush if these are invalidated.  */
 318 static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
 319                                target_ulong size)
 320 {
 321     target_ulong mask = ~(size - 1);
 322
 323     if (env->tlb_flush_addr == (target_ulong)-1) {
 324         env->tlb_flush_addr = vaddr & mask;
 325         env->tlb_flush_mask = mask;
 326         return;
 327     }
 328     /* Extend the existing region to include the new page.
 329        This is a compromise between unnecessary flushes and the cost
 330        of maintaining a full variable size TLB.  */
 331     mask &= env->tlb_flush_mask;
 332     while (((env->tlb_flush_addr ^ vaddr) & mask) != 0) {
 333         mask <<= 1;
 334     }
 335     env->tlb_flush_addr &= mask;
 336     env->tlb_flush_mask = mask;
 337 }
 338
 339 /* Add a new TLB entry. At most one entry for a given virtual address
 340  * is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
 341  * supplied size is only used by tlb_flush_page.
 342  *
 343  * Called from TCG-generated code, which is under an RCU read-side
 344  * critical section.
 345  */
 346 void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
 347                              hwaddr paddr, MemTxAttrs attrs, int prot,
 348                              int mmu_idx, target_ulong size)
 349 {
 350     CPUArchState *env = cpu->env_ptr;
 351     MemoryRegionSection *section;
 352     unsigned int index;
 353     target_ulong address;
 354     target_ulong code_address;
 355     uintptr_t addend;
 356     CPUTLBEntry *te;
 357     hwaddr iotlb, xlat, sz;
 358     unsigned vidx = env->vtlb_index++ % CPU_VTLB_SIZE;
 359     int asidx = cpu_asidx_from_attrs(cpu, attrs);
 360
 361     assert(size >= TARGET_PAGE_SIZE);
 362     if (size != TARGET_PAGE_SIZE) {
 363         tlb_add_large_page(env, vaddr, size);
 364     }
 365
 366     sz = size;
 367     section = address_space_translate_for_iotlb(cpu, asidx, paddr, &xlat, &sz);
 368     assert(sz >= TARGET_PAGE_SIZE);
 369
 370 #if defined(DEBUG_TLB)
 371     qemu_log_mask(CPU_LOG_MMU,
 372            "tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
 373            " prot=%x idx=%d\n",
 374            vaddr, paddr, prot, mmu_idx);
 375 #endif
 376
 377     address = vaddr;
 378     if (!memory_region_is_ram(section->mr) && !memory_region_is_romd(section->mr)) {
 379         /* IO memory case */
 380         address |= TLB_MMIO;
 381         addend = 0;
 382     } else {
 383         /* TLB_MMIO for rom/romd handled below */
 384         addend = (uintptr_t)memory_region_get_ram_ptr(section->mr) + xlat;
 385     }
 386
 387     code_address = address;
 388     iotlb = memory_region_section_get_iotlb(cpu, section, vaddr, paddr, xlat,
 389                                             prot, &address);
 390
 391     index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 392     te = &env->tlb_table[mmu_idx][index];
 393
 394     /* do not discard the translation in te, evict it into a victim tlb */
 395     env->tlb_v_table[mmu_idx][vidx] = *te;
 396     env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
 397
 398     /* refill the tlb */
 399     env->iotlb[mmu_idx][index].addr = iotlb - vaddr;
 400     env->iotlb[mmu_idx][index].attrs = attrs;
 401     te->addend = addend - vaddr;
 402     if (prot & PAGE_READ) {
 403         te->addr_read = address;
 404     } else {
 405         te->addr_read = -1;
 406     }
 407
 408     if (prot & PAGE_EXEC) {
 409         te->addr_code = code_address;
 410     } else {
 411         te->addr_code = -1;
 412     }
 413     if (prot & PAGE_WRITE) {
 414         if ((memory_region_is_ram(section->mr) && section->readonly)
 415             || memory_region_is_romd(section->mr)) {
 416             /* Write access calls the I/O callback.  */
 417             te->addr_write = address | TLB_MMIO;
 418         } else if (memory_region_is_ram(section->mr)
 419                    && cpu_physical_memory_is_clean(section->mr->ram_addr
 420                                                    + xlat)) {
 421             te->addr_write = address | TLB_NOTDIRTY;
 422         } else {
 423             te->addr_write = address;
 424         }
 425     } else {
 426         te->addr_write = -1;
 427     }
 428 }
 429
 430 /* Add a new TLB entry, but without specifying the memory
 431  * transaction attributes to be used.
 432  */
 433 void tlb_set_page(CPUState *cpu, target_ulong vaddr,
 434                   hwaddr paddr, int prot,
 435                   int mmu_idx, target_ulong size)
 436 {
 437     tlb_set_page_with_attrs(cpu, vaddr, paddr, MEMTXATTRS_UNSPECIFIED,
 438                             prot, mmu_idx, size);
 439 }
 440
 441 /* NOTE: this function can trigger an exception */
 442 /* NOTE2: the returned address is not exactly the physical address: it
 443  * is actually a ram_addr_t (in system mode; the user mode emulation
 444  * version of this function returns a guest virtual address).
 445  */
 446 tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 447 {
 448     int mmu_idx, page_index, pd;
 449     void *p;
 450     MemoryRegion *mr;
 451     CPUState *cpu = ENV_GET_CPU(env1);
 452     CPUIOTLBEntry *iotlbentry;
 453
 454     page_index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
 455     mmu_idx = cpu_mmu_index(env1, true);
 456     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
 457                  (addr & TARGET_PAGE_MASK))) {
 458         cpu_ldub_code(env1, addr);
 459     }
 460     iotlbentry = &env1->iotlb[mmu_idx][page_index];
 461     pd = iotlbentry->addr & ~TARGET_PAGE_MASK;
 462     mr = iotlb_to_region(cpu, pd, iotlbentry->attrs);
 463     if (memory_region_is_unassigned(mr)) {
 464         CPUClass *cc = CPU_GET_CLASS(cpu);
 465
 466         if (cc->do_unassigned_access) {
 467             cc->do_unassigned_access(cpu, addr, false, true, 0, 4);
 468         } else {
 469             cpu_abort(cpu, "Trying to execute code outside RAM or ROM at 0x"
 470                       TARGET_FMT_lx "\n", addr);
 471         }
 472     }
 473     p = (void *)((uintptr_t)addr + env1->tlb_table[mmu_idx][page_index].addend);
 474     return qemu_ram_addr_from_host_nofail(p);
 475 }
 476
 477 #define MMUSUFFIX _mmu
 478
 479 #define SHIFT 0
 480 #include "softmmu_template.h"
 481
 482 #define SHIFT 1
 483 #include "softmmu_template.h"
 484
 485 #define SHIFT 2
 486 #include "softmmu_template.h"
 487
 488 #define SHIFT 3
 489 #include "softmmu_template.h"
 490 #undef MMUSUFFIX
 491
 492 #define MMUSUFFIX _cmmu
 493 #undef GETPC_ADJ
 494 #define GETPC_ADJ 0
 495 #undef GETRA
 496 #define GETRA() ((uintptr_t)0)
 497 #define SOFTMMU_CODE_ACCESS
 498
 499 #define SHIFT 0
 500 #include "softmmu_template.h"
 501
 502 #define SHIFT 1
 503 #include "softmmu_template.h"
 504
 505 #define SHIFT 2
 506 #include "softmmu_template.h"
 507
 508 #define SHIFT 3
 509 #include "softmmu_template.h"