target/riscv/helper.c

   1 /*
   2  * RISC-V emulation helpers for qemu.
   3  *
   4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
   5  * Copyright (c) 2017-2018 SiFive, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2 or later, as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along with
  17  * this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "cpu.h"
  23 #include "exec/exec-all.h"
  24 #include "tcg-op.h"
  25
  26 #define RISCV_DEBUG_INTERRUPT 0
  27
  28 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
  29 {
  30 #ifdef CONFIG_USER_ONLY
  31     return 0;
  32 #else
  33     return env->priv;
  34 #endif
  35 }
  36
  37 #ifndef CONFIG_USER_ONLY
  38 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
  39 {
  40     target_ulong mstatus_mie = get_field(env->mstatus, MSTATUS_MIE);
  41     target_ulong mstatus_sie = get_field(env->mstatus, MSTATUS_SIE);
  42     target_ulong pending = atomic_read(&env->mip) & env->mie;
  43     target_ulong mie = env->priv < PRV_M || (env->priv == PRV_M && mstatus_mie);
  44     target_ulong sie = env->priv < PRV_S || (env->priv == PRV_S && mstatus_sie);
  45     target_ulong irqs = (pending & ~env->mideleg & -mie) |
  46                         (pending &  env->mideleg & -sie);
  47
  48     if (irqs) {
  49         return ctz64(irqs); /* since non-zero */
  50     } else {
  51         return EXCP_NONE; /* indicates no pending interrupt */
  52     }
  53 }
  54 #endif
  55
  56 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
  57 {
  58 #if !defined(CONFIG_USER_ONLY)
  59     if (interrupt_request & CPU_INTERRUPT_HARD) {
  60         RISCVCPU *cpu = RISCV_CPU(cs);
  61         CPURISCVState *env = &cpu->env;
  62         int interruptno = riscv_cpu_local_irq_pending(env);
  63         if (interruptno >= 0) {
  64             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
  65             riscv_cpu_do_interrupt(cs);
  66             return true;
  67         }
  68     }
  69 #endif
  70     return false;
  71 }
  72
  73 #if !defined(CONFIG_USER_ONLY)
  74
  75 /* get_physical_address - get the physical address for this virtual address
  76  *
  77  * Do a page table walk to obtain the physical address corresponding to a
  78  * virtual address. Returns 0 if the translation was successful
  79  *
  80  * Adapted from Spike's mmu_t::translate and mmu_t::walk
  81  *
  82  */
  83 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
  84                                 int *prot, target_ulong addr,
  85                                 int access_type, int mmu_idx)
  86 {
  87     /* NOTE: the env->pc value visible here will not be
  88      * correct, but the value visible to the exception handler
  89      * (riscv_cpu_do_interrupt) is correct */
  90
  91     int mode = mmu_idx;
  92
  93     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
  94         if (get_field(env->mstatus, MSTATUS_MPRV)) {
  95             mode = get_field(env->mstatus, MSTATUS_MPP);
  96         }
  97     }
  98
  99     if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) {
 100         *physical = addr;
 101         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 102         return TRANSLATE_SUCCESS;
 103     }
 104
 105     *prot = 0;
 106
 107     target_ulong base;
 108     int levels, ptidxbits, ptesize, vm, sum;
 109     int mxr = get_field(env->mstatus, MSTATUS_MXR);
 110
 111     if (env->priv_ver >= PRIV_VERSION_1_10_0) {
 112         base = get_field(env->satp, SATP_PPN) << PGSHIFT;
 113         sum = get_field(env->mstatus, MSTATUS_SUM);
 114         vm = get_field(env->satp, SATP_MODE);
 115         switch (vm) {
 116         case VM_1_10_SV32:
 117           levels = 2; ptidxbits = 10; ptesize = 4; break;
 118         case VM_1_10_SV39:
 119           levels = 3; ptidxbits = 9; ptesize = 8; break;
 120         case VM_1_10_SV48:
 121           levels = 4; ptidxbits = 9; ptesize = 8; break;
 122         case VM_1_10_SV57:
 123           levels = 5; ptidxbits = 9; ptesize = 8; break;
 124         case VM_1_10_MBARE:
 125             *physical = addr;
 126             *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 127             return TRANSLATE_SUCCESS;
 128         default:
 129           g_assert_not_reached();
 130         }
 131     } else {
 132         base = env->sptbr << PGSHIFT;
 133         sum = !get_field(env->mstatus, MSTATUS_PUM);
 134         vm = get_field(env->mstatus, MSTATUS_VM);
 135         switch (vm) {
 136         case VM_1_09_SV32:
 137           levels = 2; ptidxbits = 10; ptesize = 4; break;
 138         case VM_1_09_SV39:
 139           levels = 3; ptidxbits = 9; ptesize = 8; break;
 140         case VM_1_09_SV48:
 141           levels = 4; ptidxbits = 9; ptesize = 8; break;
 142         case VM_1_09_MBARE:
 143             *physical = addr;
 144             *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 145             return TRANSLATE_SUCCESS;
 146         default:
 147           g_assert_not_reached();
 148         }
 149     }
 150
 151     CPUState *cs = CPU(riscv_env_get_cpu(env));
 152     int va_bits = PGSHIFT + levels * ptidxbits;
 153     target_ulong mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
 154     target_ulong masked_msbs = (addr >> (va_bits - 1)) & mask;
 155     if (masked_msbs != 0 && masked_msbs != mask) {
 156         return TRANSLATE_FAIL;
 157     }
 158
 159     int ptshift = (levels - 1) * ptidxbits;
 160     int i;
 161
 162 #if !TCG_OVERSIZED_GUEST
 163 restart:
 164 #endif
 165     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
 166         target_ulong idx = (addr >> (PGSHIFT + ptshift)) &
 167                            ((1 << ptidxbits) - 1);
 168
 169         /* check that physical address of PTE is legal */
 170         target_ulong pte_addr = base + idx * ptesize;
 171 #if defined(TARGET_RISCV32)
 172         target_ulong pte = ldl_phys(cs->as, pte_addr);
 173 #elif defined(TARGET_RISCV64)
 174         target_ulong pte = ldq_phys(cs->as, pte_addr);
 175 #endif
 176         target_ulong ppn = pte >> PTE_PPN_SHIFT;
 177
 178         if (!(pte & PTE_V)) {
 179             /* Invalid PTE */
 180             return TRANSLATE_FAIL;
 181         } else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
 182             /* Inner PTE, continue walking */
 183             base = ppn << PGSHIFT;
 184         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
 185             /* Reserved leaf PTE flags: PTE_W */
 186             return TRANSLATE_FAIL;
 187         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
 188             /* Reserved leaf PTE flags: PTE_W + PTE_X */
 189             return TRANSLATE_FAIL;
 190         } else if ((pte & PTE_U) && ((mode != PRV_U) &&
 191                    (!sum || access_type == MMU_INST_FETCH))) {
 192             /* User PTE flags when not U mode and mstatus.SUM is not set,
 193                or the access type is an instruction fetch */
 194             return TRANSLATE_FAIL;
 195         } else if (!(pte & PTE_U) && (mode != PRV_S)) {
 196             /* Supervisor PTE flags when not S mode */
 197             return TRANSLATE_FAIL;
 198         } else if (ppn & ((1ULL << ptshift) - 1)) {
 199             /* Misaligned PPN */
 200             return TRANSLATE_FAIL;
 201         } else if (access_type == MMU_DATA_LOAD && !((pte & PTE_R) ||
 202                    ((pte & PTE_X) && mxr))) {
 203             /* Read access check failed */
 204             return TRANSLATE_FAIL;
 205         } else if (access_type == MMU_DATA_STORE && !(pte & PTE_W)) {
 206             /* Write access check failed */
 207             return TRANSLATE_FAIL;
 208         } else if (access_type == MMU_INST_FETCH && !(pte & PTE_X)) {
 209             /* Fetch access check failed */
 210             return TRANSLATE_FAIL;
 211         } else {
 212             /* if necessary, set accessed and dirty bits. */
 213             target_ulong updated_pte = pte | PTE_A |
 214                 (access_type == MMU_DATA_STORE ? PTE_D : 0);
 215
 216             /* Page table updates need to be atomic with MTTCG enabled */
 217             if (updated_pte != pte) {
 218                 /*
 219                  * - if accessed or dirty bits need updating, and the PTE is
 220                  *   in RAM, then we do so atomically with a compare and swap.
 221                  * - if the PTE is in IO space or ROM, then it can't be updated
 222                  *   and we return TRANSLATE_FAIL.
 223                  * - if the PTE changed by the time we went to update it, then
 224                  *   it is no longer valid and we must re-walk the page table.
 225                  */
 226                 MemoryRegion *mr;
 227                 hwaddr l = sizeof(target_ulong), addr1;
 228                 mr = address_space_translate(cs->as, pte_addr,
 229                     &addr1, &l, false, MEMTXATTRS_UNSPECIFIED);
 230                 if (memory_region_is_ram(mr)) {
 231                     target_ulong *pte_pa =
 232                         qemu_map_ram_ptr(mr->ram_block, addr1);
 233 #if TCG_OVERSIZED_GUEST
 234                     /* MTTCG is not enabled on oversized TCG guests so
 235                      * page table updates do not need to be atomic */
 236                     *pte_pa = pte = updated_pte;
 237 #else
 238                     target_ulong old_pte =
 239                         atomic_cmpxchg(pte_pa, pte, updated_pte);
 240                     if (old_pte != pte) {
 241                         goto restart;
 242                     } else {
 243                         pte = updated_pte;
 244                     }
 245 #endif
 246                 } else {
 247                     /* misconfigured PTE in ROM (AD bits are not preset) or
 248                      * PTE is in IO space and can't be updated atomically */
 249                     return TRANSLATE_FAIL;
 250                 }
 251             }
 252
 253             /* for superpage mappings, make a fake leaf PTE for the TLB's
 254                benefit. */
 255             target_ulong vpn = addr >> PGSHIFT;
 256             *physical = (ppn | (vpn & ((1L << ptshift) - 1))) << PGSHIFT;
 257
 258             /* set permissions on the TLB entry */
 259             if ((pte & PTE_R) || ((pte & PTE_X) && mxr)) {
 260                 *prot |= PAGE_READ;
 261             }
 262             if ((pte & PTE_X)) {
 263                 *prot |= PAGE_EXEC;
 264             }
 265             /* add write permission on stores or if the page is already dirty,
 266                so that we TLB miss on later writes to update the dirty bit */
 267             if ((pte & PTE_W) &&
 268                     (access_type == MMU_DATA_STORE || (pte & PTE_D))) {
 269                 *prot |= PAGE_WRITE;
 270             }
 271             return TRANSLATE_SUCCESS;
 272         }
 273     }
 274     return TRANSLATE_FAIL;
 275 }
 276
 277 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
 278                                 MMUAccessType access_type)
 279 {
 280     CPUState *cs = CPU(riscv_env_get_cpu(env));
 281     int page_fault_exceptions =
 282         (env->priv_ver >= PRIV_VERSION_1_10_0) &&
 283         get_field(env->satp, SATP_MODE) != VM_1_10_MBARE;
 284     switch (access_type) {
 285     case MMU_INST_FETCH:
 286         cs->exception_index = page_fault_exceptions ?
 287             RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
 288         break;
 289     case MMU_DATA_LOAD:
 290         cs->exception_index = page_fault_exceptions ?
 291             RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
 292         break;
 293     case MMU_DATA_STORE:
 294         cs->exception_index = page_fault_exceptions ?
 295             RISCV_EXCP_STORE_PAGE_FAULT : RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 296         break;
 297     default:
 298         g_assert_not_reached();
 299     }
 300     env->badaddr = address;
 301 }
 302
 303 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
 304 {
 305     RISCVCPU *cpu = RISCV_CPU(cs);
 306     hwaddr phys_addr;
 307     int prot;
 308     int mmu_idx = cpu_mmu_index(&cpu->env, false);
 309
 310     if (get_physical_address(&cpu->env, &phys_addr, &prot, addr, 0, mmu_idx)) {
 311         return -1;
 312     }
 313     return phys_addr;
 314 }
 315
 316 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
 317                                    MMUAccessType access_type, int mmu_idx,
 318                                    uintptr_t retaddr)
 319 {
 320     RISCVCPU *cpu = RISCV_CPU(cs);
 321     CPURISCVState *env = &cpu->env;
 322     switch (access_type) {
 323     case MMU_INST_FETCH:
 324         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
 325         break;
 326     case MMU_DATA_LOAD:
 327         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
 328         break;
 329     case MMU_DATA_STORE:
 330         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
 331         break;
 332     default:
 333         g_assert_not_reached();
 334     }
 335     env->badaddr = addr;
 336     do_raise_exception_err(env, cs->exception_index, retaddr);
 337 }
 338
 339 /* called by qemu's softmmu to fill the qemu tlb */
 340 void tlb_fill(CPUState *cs, target_ulong addr, int size,
 341         MMUAccessType access_type, int mmu_idx, uintptr_t retaddr)
 342 {
 343     int ret;
 344     ret = riscv_cpu_handle_mmu_fault(cs, addr, size, access_type, mmu_idx);
 345     if (ret == TRANSLATE_FAIL) {
 346         RISCVCPU *cpu = RISCV_CPU(cs);
 347         CPURISCVState *env = &cpu->env;
 348         do_raise_exception_err(env, cs->exception_index, retaddr);
 349     }
 350 }
 351
 352 #endif
 353
 354 int riscv_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int size,
 355         int rw, int mmu_idx)
 356 {
 357     RISCVCPU *cpu = RISCV_CPU(cs);
 358     CPURISCVState *env = &cpu->env;
 359 #if !defined(CONFIG_USER_ONLY)
 360     hwaddr pa = 0;
 361     int prot;
 362 #endif
 363     int ret = TRANSLATE_FAIL;
 364
 365     qemu_log_mask(CPU_LOG_MMU,
 366             "%s pc " TARGET_FMT_lx " ad %" VADDR_PRIx " rw %d mmu_idx \
 367              %d\n", __func__, env->pc, address, rw, mmu_idx);
 368
 369 #if !defined(CONFIG_USER_ONLY)
 370     ret = get_physical_address(env, &pa, &prot, address, rw, mmu_idx);
 371     qemu_log_mask(CPU_LOG_MMU,
 372             "%s address=%" VADDR_PRIx " ret %d physical " TARGET_FMT_plx
 373              " prot %d\n", __func__, address, ret, pa, prot);
 374     if (!pmp_hart_has_privs(env, pa, TARGET_PAGE_SIZE, 1 << rw)) {
 375         ret = TRANSLATE_FAIL;
 376     }
 377     if (ret == TRANSLATE_SUCCESS) {
 378         tlb_set_page(cs, address & TARGET_PAGE_MASK, pa & TARGET_PAGE_MASK,
 379                      prot, mmu_idx, TARGET_PAGE_SIZE);
 380     } else if (ret == TRANSLATE_FAIL) {
 381         raise_mmu_exception(env, address, rw);
 382     }
 383 #else
 384     switch (rw) {
 385     case MMU_INST_FETCH:
 386         cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
 387         break;
 388     case MMU_DATA_LOAD:
 389         cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
 390         break;
 391     case MMU_DATA_STORE:
 392         cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
 393         break;
 394     }
 395 #endif
 396     return ret;
 397 }
 398
 399 /*
 400  * Handle Traps
 401  *
 402  * Adapted from Spike's processor_t::take_trap.
 403  *
 404  */
 405 void riscv_cpu_do_interrupt(CPUState *cs)
 406 {
 407 #if !defined(CONFIG_USER_ONLY)
 408
 409     RISCVCPU *cpu = RISCV_CPU(cs);
 410     CPURISCVState *env = &cpu->env;
 411
 412     if (RISCV_DEBUG_INTERRUPT) {
 413         int log_cause = cs->exception_index & RISCV_EXCP_INT_MASK;
 414         if (cs->exception_index & RISCV_EXCP_INT_FLAG) {
 415             qemu_log_mask(LOG_TRACE, "core   0: trap %s, epc 0x" TARGET_FMT_lx,
 416                 riscv_intr_names[log_cause], env->pc);
 417         } else {
 418             qemu_log_mask(LOG_TRACE, "core   0: intr %s, epc 0x" TARGET_FMT_lx,
 419                 riscv_excp_names[log_cause], env->pc);
 420         }
 421     }
 422
 423     target_ulong fixed_cause = 0;
 424     if (cs->exception_index & (RISCV_EXCP_INT_FLAG)) {
 425         /* hacky for now. the MSB (bit 63) indicates interrupt but cs->exception
 426            index is only 32 bits wide */
 427         fixed_cause = cs->exception_index & RISCV_EXCP_INT_MASK;
 428         fixed_cause |= ((target_ulong)1) << (TARGET_LONG_BITS - 1);
 429     } else {
 430         /* fixup User ECALL -> correct priv ECALL */
 431         if (cs->exception_index == RISCV_EXCP_U_ECALL) {
 432             switch (env->priv) {
 433             case PRV_U:
 434                 fixed_cause = RISCV_EXCP_U_ECALL;
 435                 break;
 436             case PRV_S:
 437                 fixed_cause = RISCV_EXCP_S_ECALL;
 438                 break;
 439             case PRV_H:
 440                 fixed_cause = RISCV_EXCP_H_ECALL;
 441                 break;
 442             case PRV_M:
 443                 fixed_cause = RISCV_EXCP_M_ECALL;
 444                 break;
 445             }
 446         } else {
 447             fixed_cause = cs->exception_index;
 448         }
 449     }
 450
 451     target_ulong backup_epc = env->pc;
 452
 453     target_ulong bit = fixed_cause;
 454     target_ulong deleg = env->medeleg;
 455
 456     int hasbadaddr =
 457         (fixed_cause == RISCV_EXCP_INST_ADDR_MIS) ||
 458         (fixed_cause == RISCV_EXCP_INST_ACCESS_FAULT) ||
 459         (fixed_cause == RISCV_EXCP_LOAD_ADDR_MIS) ||
 460         (fixed_cause == RISCV_EXCP_STORE_AMO_ADDR_MIS) ||
 461         (fixed_cause == RISCV_EXCP_LOAD_ACCESS_FAULT) ||
 462         (fixed_cause == RISCV_EXCP_STORE_AMO_ACCESS_FAULT) ||
 463         (fixed_cause == RISCV_EXCP_INST_PAGE_FAULT) ||
 464         (fixed_cause == RISCV_EXCP_LOAD_PAGE_FAULT) ||
 465         (fixed_cause == RISCV_EXCP_STORE_PAGE_FAULT);
 466
 467     if (bit & ((target_ulong)1 << (TARGET_LONG_BITS - 1))) {
 468         deleg = env->mideleg;
 469         bit &= ~((target_ulong)1 << (TARGET_LONG_BITS - 1));
 470     }
 471
 472     if (env->priv <= PRV_S && bit < 64 && ((deleg >> bit) & 1)) {
 473         /* handle the trap in S-mode */
 474         /* No need to check STVEC for misaligned - lower 2 bits cannot be set */
 475         env->pc = env->stvec;
 476         env->scause = fixed_cause;
 477         env->sepc = backup_epc;
 478
 479         if (hasbadaddr) {
 480             if (RISCV_DEBUG_INTERRUPT) {
 481                 qemu_log_mask(LOG_TRACE, "core " TARGET_FMT_ld
 482                     ": badaddr 0x" TARGET_FMT_lx, env->mhartid, env->badaddr);
 483             }
 484             env->sbadaddr = env->badaddr;
 485         } else {
 486             /* otherwise we must clear sbadaddr/stval
 487              * todo: support populating stval on illegal instructions */
 488             env->sbadaddr = 0;
 489         }
 490
 491         target_ulong s = env->mstatus;
 492         s = set_field(s, MSTATUS_SPIE, env->priv_ver >= PRIV_VERSION_1_10_0 ?
 493             get_field(s, MSTATUS_SIE) : get_field(s, MSTATUS_UIE << env->priv));
 494         s = set_field(s, MSTATUS_SPP, env->priv);
 495         s = set_field(s, MSTATUS_SIE, 0);
 496         csr_write_helper(env, s, CSR_MSTATUS);
 497         riscv_set_mode(env, PRV_S);
 498     } else {
 499         /* No need to check MTVEC for misaligned - lower 2 bits cannot be set */
 500         env->pc = env->mtvec;
 501         env->mepc = backup_epc;
 502         env->mcause = fixed_cause;
 503
 504         if (hasbadaddr) {
 505             if (RISCV_DEBUG_INTERRUPT) {
 506                 qemu_log_mask(LOG_TRACE, "core " TARGET_FMT_ld
 507                     ": badaddr 0x" TARGET_FMT_lx, env->mhartid, env->badaddr);
 508             }
 509             env->mbadaddr = env->badaddr;
 510         } else {
 511             /* otherwise we must clear mbadaddr/mtval
 512              * todo: support populating mtval on illegal instructions */
 513             env->mbadaddr = 0;
 514         }
 515
 516         target_ulong s = env->mstatus;
 517         s = set_field(s, MSTATUS_MPIE, env->priv_ver >= PRIV_VERSION_1_10_0 ?
 518             get_field(s, MSTATUS_MIE) : get_field(s, MSTATUS_UIE << env->priv));
 519         s = set_field(s, MSTATUS_MPP, env->priv);
 520         s = set_field(s, MSTATUS_MIE, 0);
 521         csr_write_helper(env, s, CSR_MSTATUS);
 522         riscv_set_mode(env, PRV_M);
 523     }
 524     /* TODO yield load reservation  */
 525 #endif
 526     cs->exception_index = EXCP_NONE; /* mark handled to qemu */
 527 }