target/riscv/cpu_helper.c

   1 /*
   2  * RISC-V CPU helpers for qemu.
   3  *
   4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
   5  * Copyright (c) 2017-2018 SiFive, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2 or later, as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along with
  17  * this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "qemu/main-loop.h"
  23 #include "cpu.h"
  24 #include "exec/exec-all.h"
  25 #include "tcg/tcg-op.h"
  26 #include "trace.h"
  27
  28 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
  29 {
  30 #ifdef CONFIG_USER_ONLY
  31     return 0;
  32 #else
  33     return env->priv;
  34 #endif
  35 }
  36
  37 #ifndef CONFIG_USER_ONLY
  38 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
  39 {
  40     target_ulong irqs;
  41
  42     target_ulong mstatus_mie = get_field(env->mstatus, MSTATUS_MIE);
  43     target_ulong mstatus_sie = get_field(env->mstatus, MSTATUS_SIE);
  44     target_ulong hs_mstatus_sie = get_field(env->mstatus_hs, MSTATUS_SIE);
  45
  46     target_ulong pending = env->mip & env->mie &
  47                                ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
  48     target_ulong vspending = (env->mip & env->mie &
  49                               (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP));
  50
  51     target_ulong mie    = env->priv < PRV_M ||
  52                           (env->priv == PRV_M && mstatus_mie);
  53     target_ulong sie    = env->priv < PRV_S ||
  54                           (env->priv == PRV_S && mstatus_sie);
  55     target_ulong hs_sie = env->priv < PRV_S ||
  56                           (env->priv == PRV_S && hs_mstatus_sie);
  57
  58     if (riscv_cpu_virt_enabled(env)) {
  59         target_ulong pending_hs_irq = pending & -hs_sie;
  60
  61         if (pending_hs_irq) {
  62             riscv_cpu_set_force_hs_excep(env, FORCE_HS_EXCEP);
  63             return ctz64(pending_hs_irq);
  64         }
  65
  66         pending = vspending;
  67     }
  68
  69     irqs = (pending & ~env->mideleg & -mie) | (pending &  env->mideleg & -sie);
  70
  71     if (irqs) {
  72         return ctz64(irqs); /* since non-zero */
  73     } else {
  74         return EXCP_NONE; /* indicates no pending interrupt */
  75     }
  76 }
  77 #endif
  78
  79 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
  80 {
  81 #if !defined(CONFIG_USER_ONLY)
  82     if (interrupt_request & CPU_INTERRUPT_HARD) {
  83         RISCVCPU *cpu = RISCV_CPU(cs);
  84         CPURISCVState *env = &cpu->env;
  85         int interruptno = riscv_cpu_local_irq_pending(env);
  86         if (interruptno >= 0) {
  87             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
  88             riscv_cpu_do_interrupt(cs);
  89             return true;
  90         }
  91     }
  92 #endif
  93     return false;
  94 }
  95
  96 #if !defined(CONFIG_USER_ONLY)
  97
  98 /* Return true is floating point support is currently enabled */
  99 bool riscv_cpu_fp_enabled(CPURISCVState *env)
 100 {
 101     if (env->mstatus & MSTATUS_FS) {
 102         if (riscv_cpu_virt_enabled(env) && !(env->mstatus_hs & MSTATUS_FS)) {
 103             return false;
 104         }
 105         return true;
 106     }
 107
 108     return false;
 109 }
 110
 111 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
 112 {
 113     target_ulong mstatus_mask = MSTATUS_MXR | MSTATUS_SUM | MSTATUS_FS |
 114                                 MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE;
 115     bool current_virt = riscv_cpu_virt_enabled(env);
 116
 117     g_assert(riscv_has_ext(env, RVH));
 118
 119 #if defined(TARGET_RISCV64)
 120     mstatus_mask |= MSTATUS64_UXL;
 121 #endif
 122
 123     if (current_virt) {
 124         /* Current V=1 and we are about to change to V=0 */
 125         env->vsstatus = env->mstatus & mstatus_mask;
 126         env->mstatus &= ~mstatus_mask;
 127         env->mstatus |= env->mstatus_hs;
 128
 129 #if defined(TARGET_RISCV32)
 130         env->vsstatush = env->mstatush;
 131         env->mstatush |= env->mstatush_hs;
 132 #endif
 133
 134         env->vstvec = env->stvec;
 135         env->stvec = env->stvec_hs;
 136
 137         env->vsscratch = env->sscratch;
 138         env->sscratch = env->sscratch_hs;
 139
 140         env->vsepc = env->sepc;
 141         env->sepc = env->sepc_hs;
 142
 143         env->vscause = env->scause;
 144         env->scause = env->scause_hs;
 145
 146         env->vstval = env->sbadaddr;
 147         env->sbadaddr = env->stval_hs;
 148
 149         env->vsatp = env->satp;
 150         env->satp = env->satp_hs;
 151     } else {
 152         /* Current V=0 and we are about to change to V=1 */
 153         env->mstatus_hs = env->mstatus & mstatus_mask;
 154         env->mstatus &= ~mstatus_mask;
 155         env->mstatus |= env->vsstatus;
 156
 157 #if defined(TARGET_RISCV32)
 158         env->mstatush_hs = env->mstatush;
 159         env->mstatush |= env->vsstatush;
 160 #endif
 161
 162         env->stvec_hs = env->stvec;
 163         env->stvec = env->vstvec;
 164
 165         env->sscratch_hs = env->sscratch;
 166         env->sscratch = env->vsscratch;
 167
 168         env->sepc_hs = env->sepc;
 169         env->sepc = env->vsepc;
 170
 171         env->scause_hs = env->scause;
 172         env->scause = env->vscause;
 173
 174         env->stval_hs = env->sbadaddr;
 175         env->sbadaddr = env->vstval;
 176
 177         env->satp_hs = env->satp;
 178         env->satp = env->vsatp;
 179     }
 180 }
 181
 182 bool riscv_cpu_virt_enabled(CPURISCVState *env)
 183 {
 184     if (!riscv_has_ext(env, RVH)) {
 185         return false;
 186     }
 187
 188     return get_field(env->virt, VIRT_ONOFF);
 189 }
 190
 191 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable)
 192 {
 193     if (!riscv_has_ext(env, RVH)) {
 194         return;
 195     }
 196
 197     /* Flush the TLB on all virt mode changes. */
 198     if (get_field(env->virt, VIRT_ONOFF) != enable) {
 199         tlb_flush(env_cpu(env));
 200     }
 201
 202     env->virt = set_field(env->virt, VIRT_ONOFF, enable);
 203 }
 204
 205 bool riscv_cpu_force_hs_excep_enabled(CPURISCVState *env)
 206 {
 207     if (!riscv_has_ext(env, RVH)) {
 208         return false;
 209     }
 210
 211     return get_field(env->virt, FORCE_HS_EXCEP);
 212 }
 213
 214 void riscv_cpu_set_force_hs_excep(CPURISCVState *env, bool enable)
 215 {
 216     if (!riscv_has_ext(env, RVH)) {
 217         return;
 218     }
 219
 220     env->virt = set_field(env->virt, FORCE_HS_EXCEP, enable);
 221 }
 222
 223 bool riscv_cpu_two_stage_lookup(CPURISCVState *env)
 224 {
 225     if (!riscv_has_ext(env, RVH)) {
 226         return false;
 227     }
 228
 229     return get_field(env->virt, HS_TWO_STAGE);
 230 }
 231
 232 void riscv_cpu_set_two_stage_lookup(CPURISCVState *env, bool enable)
 233 {
 234     if (!riscv_has_ext(env, RVH)) {
 235         return;
 236     }
 237
 238     env->virt = set_field(env->virt, HS_TWO_STAGE, enable);
 239 }
 240
 241 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint32_t interrupts)
 242 {
 243     CPURISCVState *env = &cpu->env;
 244     if (env->miclaim & interrupts) {
 245         return -1;
 246     } else {
 247         env->miclaim |= interrupts;
 248         return 0;
 249     }
 250 }
 251
 252 uint32_t riscv_cpu_update_mip(RISCVCPU *cpu, uint32_t mask, uint32_t value)
 253 {
 254     CPURISCVState *env = &cpu->env;
 255     CPUState *cs = CPU(cpu);
 256     uint32_t old = env->mip;
 257     bool locked = false;
 258
 259     if (!qemu_mutex_iothread_locked()) {
 260         locked = true;
 261         qemu_mutex_lock_iothread();
 262     }
 263
 264     env->mip = (env->mip & ~mask) | (value & mask);
 265
 266     if (env->mip) {
 267         cpu_interrupt(cs, CPU_INTERRUPT_HARD);
 268     } else {
 269         cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
 270     }
 271
 272     if (locked) {
 273         qemu_mutex_unlock_iothread();
 274     }
 275
 276     return old;
 277 }
 278
 279 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(void))
 280 {
 281     env->rdtime_fn = fn;
 282 }
 283
 284 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
 285 {
 286     if (newpriv > PRV_M) {
 287         g_assert_not_reached();
 288     }
 289     if (newpriv == PRV_H) {
 290         newpriv = PRV_U;
 291     }
 292     /* tlb_flush is unnecessary as mode is contained in mmu_idx */
 293     env->priv = newpriv;
 294
 295     /*
 296      * Clear the load reservation - otherwise a reservation placed in one
 297      * context/process can be used by another, resulting in an SC succeeding
 298      * incorrectly. Version 2.2 of the ISA specification explicitly requires
 299      * this behaviour, while later revisions say that the kernel "should" use
 300      * an SC instruction to force the yielding of a load reservation on a
 301      * preemptive context switch. As a result, do both.
 302      */
 303     env->load_res = -1;
 304 }
 305
 306 /* get_physical_address - get the physical address for this virtual address
 307  *
 308  * Do a page table walk to obtain the physical address corresponding to a
 309  * virtual address. Returns 0 if the translation was successful
 310  *
 311  * Adapted from Spike's mmu_t::translate and mmu_t::walk
 312  *
 313  * @env: CPURISCVState
 314  * @physical: This will be set to the calculated physical address
 315  * @prot: The returned protection attributes
 316  * @addr: The virtual address to be translated
 317  * @access_type: The type of MMU access
 318  * @mmu_idx: Indicates current privilege level
 319  * @first_stage: Are we in first stage translation?
 320  *               Second stage is used for hypervisor guest translation
 321  * @two_stage: Are we going to perform two stage translation
 322  */
 323 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
 324                                 int *prot, target_ulong addr,
 325                                 int access_type, int mmu_idx,
 326                                 bool first_stage, bool two_stage)
 327 {
 328     /* NOTE: the env->pc value visible here will not be
 329      * correct, but the value visible to the exception handler
 330      * (riscv_cpu_do_interrupt) is correct */
 331     MemTxResult res;
 332     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
 333     int mode = mmu_idx;
 334     bool use_background = false;
 335
 336     /*
 337      * Check if we should use the background registers for the two
 338      * stage translation. We don't need to check if we actually need
 339      * two stage translation as that happened before this function
 340      * was called. Background registers will be used if the guest has
 341      * forced a two stage translation to be on (in HS or M mode).
 342      */
 343     if (riscv_cpu_two_stage_lookup(env) && access_type != MMU_INST_FETCH) {
 344         use_background = true;
 345     }
 346
 347     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 348         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 349             mode = get_field(env->mstatus, MSTATUS_MPP);
 350         }
 351     }
 352
 353     if (first_stage == false) {
 354         /* We are in stage 2 translation, this is similar to stage 1. */
 355         /* Stage 2 is always taken as U-mode */
 356         mode = PRV_U;
 357     }
 358
 359     if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) {
 360         *physical = addr;
 361         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 362         return TRANSLATE_SUCCESS;
 363     }
 364
 365     *prot = 0;
 366
 367     hwaddr base;
 368     int levels, ptidxbits, ptesize, vm, sum, mxr, widened;
 369
 370     if (first_stage == true) {
 371         mxr = get_field(env->mstatus, MSTATUS_MXR);
 372     } else {
 373         mxr = get_field(env->vsstatus, MSTATUS_MXR);
 374     }
 375
 376     if (first_stage == true) {
 377         if (use_background) {
 378             base = (hwaddr)get_field(env->vsatp, SATP_PPN) << PGSHIFT;
 379             vm = get_field(env->vsatp, SATP_MODE);
 380         } else {
 381             base = (hwaddr)get_field(env->satp, SATP_PPN) << PGSHIFT;
 382             vm = get_field(env->satp, SATP_MODE);
 383         }
 384         widened = 0;
 385     } else {
 386         base = (hwaddr)get_field(env->hgatp, HGATP_PPN) << PGSHIFT;
 387         vm = get_field(env->hgatp, HGATP_MODE);
 388         widened = 2;
 389     }
 390     sum = get_field(env->mstatus, MSTATUS_SUM);
 391     switch (vm) {
 392     case VM_1_10_SV32:
 393       levels = 2; ptidxbits = 10; ptesize = 4; break;
 394     case VM_1_10_SV39:
 395       levels = 3; ptidxbits = 9; ptesize = 8; break;
 396     case VM_1_10_SV48:
 397       levels = 4; ptidxbits = 9; ptesize = 8; break;
 398     case VM_1_10_SV57:
 399       levels = 5; ptidxbits = 9; ptesize = 8; break;
 400     case VM_1_10_MBARE:
 401         *physical = addr;
 402         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 403         return TRANSLATE_SUCCESS;
 404     default:
 405       g_assert_not_reached();
 406     }
 407
 408     CPUState *cs = env_cpu(env);
 409     int va_bits = PGSHIFT + levels * ptidxbits + widened;
 410     target_ulong mask, masked_msbs;
 411
 412     if (TARGET_LONG_BITS > (va_bits - 1)) {
 413         mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
 414     } else {
 415         mask = 0;
 416     }
 417     masked_msbs = (addr >> (va_bits - 1)) & mask;
 418
 419     if (masked_msbs != 0 && masked_msbs != mask) {
 420         return TRANSLATE_FAIL;
 421     }
 422
 423     int ptshift = (levels - 1) * ptidxbits;
 424     int i;
 425
 426 #if !TCG_OVERSIZED_GUEST
 427 restart:
 428 #endif
 429     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
 430         target_ulong idx;
 431         if (i == 0) {
 432             idx = (addr >> (PGSHIFT + ptshift)) &
 433                            ((1 << (ptidxbits + widened)) - 1);
 434         } else {
 435             idx = (addr >> (PGSHIFT + ptshift)) &
 436                            ((1 << ptidxbits) - 1);
 437         }
 438
 439         /* check that physical address of PTE is legal */
 440         hwaddr pte_addr;
 441
 442         if (two_stage && first_stage) {
 443             int vbase_prot;
 444             hwaddr vbase;
 445
 446             /* Do the second stage translation on the base PTE address. */
 447             int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
 448                                                  base, MMU_DATA_LOAD,
 449                                                  mmu_idx, false, true);
 450
 451             if (vbase_ret != TRANSLATE_SUCCESS) {
 452                 return vbase_ret;
 453             }
 454
 455             pte_addr = vbase + idx * ptesize;
 456         } else {
 457             pte_addr = base + idx * ptesize;
 458         }
 459
 460         if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 461             !pmp_hart_has_privs(env, pte_addr, sizeof(target_ulong),
 462             1 << MMU_DATA_LOAD, PRV_S)) {
 463             return TRANSLATE_PMP_FAIL;
 464         }
 465
 466 #if defined(TARGET_RISCV32)
 467         target_ulong pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
 468 #elif defined(TARGET_RISCV64)
 469         target_ulong pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
 470 #endif
 471         if (res != MEMTX_OK) {
 472             return TRANSLATE_FAIL;
 473         }
 474
 475         hwaddr ppn = pte >> PTE_PPN_SHIFT;
 476
 477         if (!(pte & PTE_V)) {
 478             /* Invalid PTE */
 479             return TRANSLATE_FAIL;
 480         } else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
 481             /* Inner PTE, continue walking */
 482             base = ppn << PGSHIFT;
 483         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
 484             /* Reserved leaf PTE flags: PTE_W */
 485             return TRANSLATE_FAIL;
 486         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
 487             /* Reserved leaf PTE flags: PTE_W + PTE_X */
 488             return TRANSLATE_FAIL;
 489         } else if ((pte & PTE_U) && ((mode != PRV_U) &&
 490                    (!sum || access_type == MMU_INST_FETCH))) {
 491             /* User PTE flags when not U mode and mstatus.SUM is not set,
 492                or the access type is an instruction fetch */
 493             return TRANSLATE_FAIL;
 494         } else if (!(pte & PTE_U) && (mode != PRV_S)) {
 495             /* Supervisor PTE flags when not S mode */
 496             return TRANSLATE_FAIL;
 497         } else if (ppn & ((1ULL << ptshift) - 1)) {
 498             /* Misaligned PPN */
 499             return TRANSLATE_FAIL;
 500         } else if (access_type == MMU_DATA_LOAD && !((pte & PTE_R) ||
 501                    ((pte & PTE_X) && mxr))) {
 502             /* Read access check failed */
 503             return TRANSLATE_FAIL;
 504         } else if (access_type == MMU_DATA_STORE && !(pte & PTE_W)) {
 505             /* Write access check failed */
 506             return TRANSLATE_FAIL;
 507         } else if (access_type == MMU_INST_FETCH && !(pte & PTE_X)) {
 508             /* Fetch access check failed */
 509             return TRANSLATE_FAIL;
 510         } else {
 511             /* if necessary, set accessed and dirty bits. */
 512             target_ulong updated_pte = pte | PTE_A |
 513                 (access_type == MMU_DATA_STORE ? PTE_D : 0);
 514
 515             /* Page table updates need to be atomic with MTTCG enabled */
 516             if (updated_pte != pte) {
 517                 /*
 518                  * - if accessed or dirty bits need updating, and the PTE is
 519                  *   in RAM, then we do so atomically with a compare and swap.
 520                  * - if the PTE is in IO space or ROM, then it can't be updated
 521                  *   and we return TRANSLATE_FAIL.
 522                  * - if the PTE changed by the time we went to update it, then
 523                  *   it is no longer valid and we must re-walk the page table.
 524                  */
 525                 MemoryRegion *mr;
 526                 hwaddr l = sizeof(target_ulong), addr1;
 527                 mr = address_space_translate(cs->as, pte_addr,
 528                     &addr1, &l, false, MEMTXATTRS_UNSPECIFIED);
 529                 if (memory_region_is_ram(mr)) {
 530                     target_ulong *pte_pa =
 531                         qemu_map_ram_ptr(mr->ram_block, addr1);
 532 #if TCG_OVERSIZED_GUEST
 533                     /* MTTCG is not enabled on oversized TCG guests so
 534                      * page table updates do not need to be atomic */
 535                     *pte_pa = pte = updated_pte;
 536 #else
 537                     target_ulong old_pte =
 538                         atomic_cmpxchg(pte_pa, pte, updated_pte);
 539                     if (old_pte != pte) {
 540                         goto restart;
 541                     } else {
 542                         pte = updated_pte;
 543                     }
 544 #endif
 545                 } else {
 546                     /* misconfigured PTE in ROM (AD bits are not preset) or
 547                      * PTE is in IO space and can't be updated atomically */
 548                     return TRANSLATE_FAIL;
 549                 }
 550             }
 551
 552             /* for superpage mappings, make a fake leaf PTE for the TLB's
 553                benefit. */
 554             target_ulong vpn = addr >> PGSHIFT;
 555             *physical = ((ppn | (vpn & ((1L << ptshift) - 1))) << PGSHIFT) |
 556                         (addr & ~TARGET_PAGE_MASK);
 557
 558             /* set permissions on the TLB entry */
 559             if ((pte & PTE_R) || ((pte & PTE_X) && mxr)) {
 560                 *prot |= PAGE_READ;
 561             }
 562             if ((pte & PTE_X)) {
 563                 *prot |= PAGE_EXEC;
 564             }
 565             /* add write permission on stores or if the page is already dirty,
 566                so that we TLB miss on later writes to update the dirty bit */
 567             if ((pte & PTE_W) &&
 568                     (access_type == MMU_DATA_STORE || (pte & PTE_D))) {
 569                 *prot |= PAGE_WRITE;
 570             }
 571             return TRANSLATE_SUCCESS;
 572         }
 573     }
 574     return TRANSLATE_FAIL;
 575 }
 576
 577 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
 578                                 MMUAccessType access_type, bool pmp_violation,
 579                                 bool first_stage)
 580 {
 581     CPUState *cs = env_cpu(env);
 582     int page_fault_exceptions;
 583     if (first_stage) {
 584         page_fault_exceptions =
 585             get_field(env->satp, SATP_MODE) != VM_1_10_MBARE &&
 586             !pmp_violation;
 587     } else {
 588         page_fault_exceptions =
 589             get_field(env->hgatp, HGATP_MODE) != VM_1_10_MBARE &&
 590             !pmp_violation;
 591     }
 592     switch (access_type) {
 593     case MMU_INST_FETCH:
 594         if (riscv_cpu_virt_enabled(env) && !first_stage) {
 595             cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
 596         } else {
 597             cs->exception_index = page_fault_exceptions ?
 598                 RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
 599         }
 600         break;
 601     case MMU_DATA_LOAD:
 602         if ((riscv_cpu_virt_enabled(env) || riscv_cpu_two_stage_lookup(env)) &&
 603             !first_stage) {
 604             cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
 605         } else {
 606             cs->exception_index = page_fault_exceptions ?
 607                 RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
 608         }
 609         break;
 610     case MMU_DATA_STORE:
 611         if ((riscv_cpu_virt_enabled(env) || riscv_cpu_two_stage_lookup(env)) &&
 612             !first_stage) {
 613             cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
 614         } else {
 615             cs->exception_index = page_fault_exceptions ?
 616                 RISCV_EXCP_STORE_PAGE_FAULT : RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 617         }
 618         break;
 619     default:
 620         g_assert_not_reached();
 621     }
 622     env->badaddr = address;
 623 }
 624
 625 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
 626 {
 627     RISCVCPU *cpu = RISCV_CPU(cs);
 628     CPURISCVState *env = &cpu->env;
 629     hwaddr phys_addr;
 630     int prot;
 631     int mmu_idx = cpu_mmu_index(&cpu->env, false);
 632
 633     if (get_physical_address(env, &phys_addr, &prot, addr, 0, mmu_idx,
 634                              true, riscv_cpu_virt_enabled(env))) {
 635         return -1;
 636     }
 637
 638     if (riscv_cpu_virt_enabled(env)) {
 639         if (get_physical_address(env, &phys_addr, &prot, phys_addr,
 640                                  0, mmu_idx, false, true)) {
 641             return -1;
 642         }
 643     }
 644
 645     return phys_addr & TARGET_PAGE_MASK;
 646 }
 647
 648 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
 649                                      vaddr addr, unsigned size,
 650                                      MMUAccessType access_type,
 651                                      int mmu_idx, MemTxAttrs attrs,
 652                                      MemTxResult response, uintptr_t retaddr)
 653 {
 654     RISCVCPU *cpu = RISCV_CPU(cs);
 655     CPURISCVState *env = &cpu->env;
 656
 657     if (access_type == MMU_DATA_STORE) {
 658         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 659     } else {
 660         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
 661     }
 662
 663     env->badaddr = addr;
 664     riscv_raise_exception(&cpu->env, cs->exception_index, retaddr);
 665 }
 666
 667 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
 668                                    MMUAccessType access_type, int mmu_idx,
 669                                    uintptr_t retaddr)
 670 {
 671     RISCVCPU *cpu = RISCV_CPU(cs);
 672     CPURISCVState *env = &cpu->env;
 673     switch (access_type) {
 674     case MMU_INST_FETCH:
 675         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
 676         break;
 677     case MMU_DATA_LOAD:
 678         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
 679         break;
 680     case MMU_DATA_STORE:
 681         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
 682         break;
 683     default:
 684         g_assert_not_reached();
 685     }
 686     env->badaddr = addr;
 687     riscv_raise_exception(env, cs->exception_index, retaddr);
 688 }
 689 #endif
 690
 691 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
 692                         MMUAccessType access_type, int mmu_idx,
 693                         bool probe, uintptr_t retaddr)
 694 {
 695     RISCVCPU *cpu = RISCV_CPU(cs);
 696     CPURISCVState *env = &cpu->env;
 697 #ifndef CONFIG_USER_ONLY
 698     vaddr im_address;
 699     hwaddr pa = 0;
 700     int prot, prot2;
 701     bool pmp_violation = false;
 702     bool first_stage_error = true;
 703     int ret = TRANSLATE_FAIL;
 704     int mode = mmu_idx;
 705     target_ulong tlb_size = 0;
 706
 707     env->guest_phys_fault_addr = 0;
 708
 709     qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
 710                   __func__, address, access_type, mmu_idx);
 711
 712     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 713         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 714             mode = get_field(env->mstatus, MSTATUS_MPP);
 715         }
 716     }
 717
 718     if (riscv_has_ext(env, RVH) && env->priv == PRV_M &&
 719         access_type != MMU_INST_FETCH &&
 720         get_field(env->mstatus, MSTATUS_MPRV) &&
 721         MSTATUS_MPV_ISSET(env)) {
 722         riscv_cpu_set_two_stage_lookup(env, true);
 723     }
 724
 725     if (riscv_cpu_virt_enabled(env) ||
 726         (riscv_cpu_two_stage_lookup(env) && access_type != MMU_INST_FETCH)) {
 727         /* Two stage lookup */
 728         ret = get_physical_address(env, &pa, &prot, address, access_type,
 729                                    mmu_idx, true, true);
 730
 731         qemu_log_mask(CPU_LOG_MMU,
 732                       "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
 733                       TARGET_FMT_plx " prot %d\n",
 734                       __func__, address, ret, pa, prot);
 735
 736         if (ret != TRANSLATE_FAIL) {
 737             /* Second stage lookup */
 738             im_address = pa;
 739
 740             ret = get_physical_address(env, &pa, &prot2, im_address,
 741                                        access_type, mmu_idx, false, true);
 742
 743             qemu_log_mask(CPU_LOG_MMU,
 744                     "%s 2nd-stage address=%" VADDR_PRIx " ret %d physical "
 745                     TARGET_FMT_plx " prot %d\n",
 746                     __func__, im_address, ret, pa, prot2);
 747
 748             prot &= prot2;
 749
 750             if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 751                 (ret == TRANSLATE_SUCCESS) &&
 752                 !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
 753                 ret = TRANSLATE_PMP_FAIL;
 754             }
 755
 756             if (ret != TRANSLATE_SUCCESS) {
 757                 /*
 758                  * Guest physical address translation failed, this is a HS
 759                  * level exception
 760                  */
 761                 first_stage_error = false;
 762                 env->guest_phys_fault_addr = (im_address |
 763                                               (address &
 764                                                (TARGET_PAGE_SIZE - 1))) >> 2;
 765             }
 766         }
 767     } else {
 768         /* Single stage lookup */
 769         ret = get_physical_address(env, &pa, &prot, address, access_type,
 770                                    mmu_idx, true, false);
 771
 772         qemu_log_mask(CPU_LOG_MMU,
 773                       "%s address=%" VADDR_PRIx " ret %d physical "
 774                       TARGET_FMT_plx " prot %d\n",
 775                       __func__, address, ret, pa, prot);
 776     }
 777
 778     /* We did the two stage lookup based on MPRV, unset the lookup */
 779     if (riscv_has_ext(env, RVH) && env->priv == PRV_M &&
 780         access_type != MMU_INST_FETCH &&
 781         get_field(env->mstatus, MSTATUS_MPRV) &&
 782         MSTATUS_MPV_ISSET(env)) {
 783         riscv_cpu_set_two_stage_lookup(env, false);
 784     }
 785
 786     if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 787         (ret == TRANSLATE_SUCCESS) &&
 788         !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
 789         ret = TRANSLATE_PMP_FAIL;
 790     }
 791     if (ret == TRANSLATE_PMP_FAIL) {
 792         pmp_violation = true;
 793     }
 794
 795     if (ret == TRANSLATE_SUCCESS) {
 796         if (pmp_is_range_in_tlb(env, pa & TARGET_PAGE_MASK, &tlb_size)) {
 797             tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
 798                          prot, mmu_idx, tlb_size);
 799         } else {
 800             tlb_set_page(cs, address & TARGET_PAGE_MASK, pa & TARGET_PAGE_MASK,
 801                          prot, mmu_idx, TARGET_PAGE_SIZE);
 802         }
 803         return true;
 804     } else if (probe) {
 805         return false;
 806     } else {
 807         raise_mmu_exception(env, address, access_type, pmp_violation, first_stage_error);
 808         riscv_raise_exception(env, cs->exception_index, retaddr);
 809     }
 810
 811     return true;
 812
 813 #else
 814     switch (access_type) {
 815     case MMU_INST_FETCH:
 816         cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
 817         break;
 818     case MMU_DATA_LOAD:
 819         cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
 820         break;
 821     case MMU_DATA_STORE:
 822         cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
 823         break;
 824     default:
 825         g_assert_not_reached();
 826     }
 827     env->badaddr = address;
 828     cpu_loop_exit_restore(cs, retaddr);
 829 #endif
 830 }
 831
 832 /*
 833  * Handle Traps
 834  *
 835  * Adapted from Spike's processor_t::take_trap.
 836  *
 837  */
 838 void riscv_cpu_do_interrupt(CPUState *cs)
 839 {
 840 #if !defined(CONFIG_USER_ONLY)
 841
 842     RISCVCPU *cpu = RISCV_CPU(cs);
 843     CPURISCVState *env = &cpu->env;
 844     bool force_hs_execp = riscv_cpu_force_hs_excep_enabled(env);
 845     target_ulong s;
 846
 847     /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
 848      * so we mask off the MSB and separate into trap type and cause.
 849      */
 850     bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
 851     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
 852     target_ulong deleg = async ? env->mideleg : env->medeleg;
 853     target_ulong tval = 0;
 854     target_ulong htval = 0;
 855     target_ulong mtval2 = 0;
 856
 857     if (!async) {
 858         /* set tval to badaddr for traps with address information */
 859         switch (cause) {
 860         case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
 861         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
 862         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
 863             force_hs_execp = true;
 864             /* fallthrough */
 865         case RISCV_EXCP_INST_ADDR_MIS:
 866         case RISCV_EXCP_INST_ACCESS_FAULT:
 867         case RISCV_EXCP_LOAD_ADDR_MIS:
 868         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
 869         case RISCV_EXCP_LOAD_ACCESS_FAULT:
 870         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
 871         case RISCV_EXCP_INST_PAGE_FAULT:
 872         case RISCV_EXCP_LOAD_PAGE_FAULT:
 873         case RISCV_EXCP_STORE_PAGE_FAULT:
 874             tval = env->badaddr;
 875             break;
 876         default:
 877             break;
 878         }
 879         /* ecall is dispatched as one cause so translate based on mode */
 880         if (cause == RISCV_EXCP_U_ECALL) {
 881             assert(env->priv <= 3);
 882
 883             if (env->priv == PRV_M) {
 884                 cause = RISCV_EXCP_M_ECALL;
 885             } else if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
 886                 cause = RISCV_EXCP_VS_ECALL;
 887             } else if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) {
 888                 cause = RISCV_EXCP_S_ECALL;
 889             } else if (env->priv == PRV_U) {
 890                 cause = RISCV_EXCP_U_ECALL;
 891             }
 892         }
 893     }
 894
 895     trace_riscv_trap(env->mhartid, async, cause, env->pc, tval, cause < 23 ?
 896         (async ? riscv_intr_names : riscv_excp_names)[cause] : "(unknown)");
 897
 898     if (env->priv <= PRV_S &&
 899             cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
 900         /* handle the trap in S-mode */
 901         if (riscv_has_ext(env, RVH)) {
 902             target_ulong hdeleg = async ? env->hideleg : env->hedeleg;
 903
 904             if ((riscv_cpu_virt_enabled(env) ||
 905                  riscv_cpu_two_stage_lookup(env)) && tval) {
 906                 /*
 907                  * If we are writing a guest virtual address to stval, set
 908                  * this to 1. If we are trapping to VS we will set this to 0
 909                  * later.
 910                  */
 911                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 1);
 912             } else {
 913                 /* For other HS-mode traps, we set this to 0. */
 914                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
 915             }
 916
 917             if (riscv_cpu_virt_enabled(env) && ((hdeleg >> cause) & 1) &&
 918                 !force_hs_execp) {
 919                 /* Trap to VS mode */
 920                 /*
 921                  * See if we need to adjust cause. Yes if its VS mode interrupt
 922                  * no if hypervisor has delegated one of hs mode's interrupt
 923                  */
 924                 if (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
 925                     cause == IRQ_VS_EXT) {
 926                     cause = cause - 1;
 927                 }
 928                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
 929             } else if (riscv_cpu_virt_enabled(env)) {
 930                 /* Trap into HS mode, from virt */
 931                 riscv_cpu_swap_hypervisor_regs(env);
 932                 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
 933                                          get_field(env->mstatus, SSTATUS_SPP));
 934                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 935                                          riscv_cpu_virt_enabled(env));
 936
 937                 htval = env->guest_phys_fault_addr;
 938
 939                 riscv_cpu_set_virt_enabled(env, 0);
 940                 riscv_cpu_set_force_hs_excep(env, 0);
 941             } else {
 942                 /* Trap into HS mode */
 943                 if (!riscv_cpu_two_stage_lookup(env)) {
 944                     env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 945                                              riscv_cpu_virt_enabled(env));
 946                 }
 947                 riscv_cpu_set_two_stage_lookup(env, false);
 948                 htval = env->guest_phys_fault_addr;
 949             }
 950         }
 951
 952         s = env->mstatus;
 953         s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
 954         s = set_field(s, MSTATUS_SPP, env->priv);
 955         s = set_field(s, MSTATUS_SIE, 0);
 956         env->mstatus = s;
 957         env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
 958         env->sepc = env->pc;
 959         env->sbadaddr = tval;
 960         env->htval = htval;
 961         env->pc = (env->stvec >> 2 << 2) +
 962             ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
 963         riscv_cpu_set_mode(env, PRV_S);
 964     } else {
 965         /* handle the trap in M-mode */
 966         if (riscv_has_ext(env, RVH)) {
 967             if (riscv_cpu_virt_enabled(env)) {
 968                 riscv_cpu_swap_hypervisor_regs(env);
 969             }
 970 #ifdef TARGET_RISCV32
 971             env->mstatush = set_field(env->mstatush, MSTATUS_MPV,
 972                                        riscv_cpu_virt_enabled(env));
 973             if (riscv_cpu_virt_enabled(env) && tval) {
 974                 env->mstatush = set_field(env->mstatush, MSTATUS_GVA, 1);
 975             }
 976 #else
 977             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
 978                                       riscv_cpu_virt_enabled(env));
 979             if (riscv_cpu_virt_enabled(env) && tval) {
 980                 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
 981             }
 982 #endif
 983
 984             mtval2 = env->guest_phys_fault_addr;
 985
 986             /* Trapping to M mode, virt is disabled */
 987             riscv_cpu_set_virt_enabled(env, 0);
 988             riscv_cpu_set_force_hs_excep(env, 0);
 989         }
 990
 991         s = env->mstatus;
 992         s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
 993         s = set_field(s, MSTATUS_MPP, env->priv);
 994         s = set_field(s, MSTATUS_MIE, 0);
 995         env->mstatus = s;
 996         env->mcause = cause | ~(((target_ulong)-1) >> async);
 997         env->mepc = env->pc;
 998         env->mbadaddr = tval;
 999         env->mtval2 = mtval2;
1000         env->pc = (env->mtvec >> 2 << 2) +
1001             ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
1002         riscv_cpu_set_mode(env, PRV_M);
1003     }
1004
1005     /* NOTE: it is not necessary to yield load reservations here. It is only
1006      * necessary for an SC from "another hart" to cause a load reservation
1007      * to be yielded. Refer to the memory consistency model section of the
1008      * RISC-V ISA Specification.
1009      */
1010
1011 #endif
1012     cs->exception_index = EXCP_NONE; /* mark handled to qemu */
1013 }