target/riscv/cpu_helper.c

   1 /*
   2  * RISC-V CPU helpers for qemu.
   3  *
   4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
   5  * Copyright (c) 2017-2018 SiFive, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2 or later, as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along with
  17  * this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "qemu/main-loop.h"
  23 #include "cpu.h"
  24 #include "exec/exec-all.h"
  25 #include "tcg/tcg-op.h"
  26 #include "trace.h"
  27
  28 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
  29 {
  30 #ifdef CONFIG_USER_ONLY
  31     return 0;
  32 #else
  33     return env->priv;
  34 #endif
  35 }
  36
  37 #ifndef CONFIG_USER_ONLY
  38 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
  39 {
  40     target_ulong irqs;
  41
  42     target_ulong mstatus_mie = get_field(env->mstatus, MSTATUS_MIE);
  43     target_ulong mstatus_sie = get_field(env->mstatus, MSTATUS_SIE);
  44     target_ulong hs_mstatus_sie = get_field(env->mstatus_hs, MSTATUS_SIE);
  45
  46     target_ulong pending = env->mip & env->mie &
  47                                ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
  48     target_ulong vspending = (env->mip & env->mie &
  49                               (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP));
  50
  51     target_ulong mie    = env->priv < PRV_M ||
  52                           (env->priv == PRV_M && mstatus_mie);
  53     target_ulong sie    = env->priv < PRV_S ||
  54                           (env->priv == PRV_S && mstatus_sie);
  55     target_ulong hs_sie = env->priv < PRV_S ||
  56                           (env->priv == PRV_S && hs_mstatus_sie);
  57
  58     if (riscv_cpu_virt_enabled(env)) {
  59         target_ulong pending_hs_irq = pending & -hs_sie;
  60
  61         if (pending_hs_irq) {
  62             riscv_cpu_set_force_hs_excep(env, FORCE_HS_EXCEP);
  63             return ctz64(pending_hs_irq);
  64         }
  65
  66         pending = vspending;
  67     }
  68
  69     irqs = (pending & ~env->mideleg & -mie) | (pending &  env->mideleg & -sie);
  70
  71     if (irqs) {
  72         return ctz64(irqs); /* since non-zero */
  73     } else {
  74         return EXCP_NONE; /* indicates no pending interrupt */
  75     }
  76 }
  77 #endif
  78
  79 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
  80 {
  81 #if !defined(CONFIG_USER_ONLY)
  82     if (interrupt_request & CPU_INTERRUPT_HARD) {
  83         RISCVCPU *cpu = RISCV_CPU(cs);
  84         CPURISCVState *env = &cpu->env;
  85         int interruptno = riscv_cpu_local_irq_pending(env);
  86         if (interruptno >= 0) {
  87             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
  88             riscv_cpu_do_interrupt(cs);
  89             return true;
  90         }
  91     }
  92 #endif
  93     return false;
  94 }
  95
  96 #if !defined(CONFIG_USER_ONLY)
  97
  98 /* Return true is floating point support is currently enabled */
  99 bool riscv_cpu_fp_enabled(CPURISCVState *env)
 100 {
 101     if (env->mstatus & MSTATUS_FS) {
 102         if (riscv_cpu_virt_enabled(env) && !(env->mstatus_hs & MSTATUS_FS)) {
 103             return false;
 104         }
 105         return true;
 106     }
 107
 108     return false;
 109 }
 110
 111 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
 112 {
 113     target_ulong mstatus_mask = MSTATUS_MXR | MSTATUS_SUM | MSTATUS_FS |
 114                                 MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE;
 115     bool current_virt = riscv_cpu_virt_enabled(env);
 116
 117     g_assert(riscv_has_ext(env, RVH));
 118
 119 #if defined(TARGET_RISCV64)
 120     mstatus_mask |= MSTATUS64_UXL;
 121 #endif
 122
 123     if (current_virt) {
 124         /* Current V=1 and we are about to change to V=0 */
 125         env->vsstatus = env->mstatus & mstatus_mask;
 126         env->mstatus &= ~mstatus_mask;
 127         env->mstatus |= env->mstatus_hs;
 128
 129 #if defined(TARGET_RISCV32)
 130         env->vsstatush = env->mstatush;
 131         env->mstatush |= env->mstatush_hs;
 132 #endif
 133
 134         env->vstvec = env->stvec;
 135         env->stvec = env->stvec_hs;
 136
 137         env->vsscratch = env->sscratch;
 138         env->sscratch = env->sscratch_hs;
 139
 140         env->vsepc = env->sepc;
 141         env->sepc = env->sepc_hs;
 142
 143         env->vscause = env->scause;
 144         env->scause = env->scause_hs;
 145
 146         env->vstval = env->sbadaddr;
 147         env->sbadaddr = env->stval_hs;
 148
 149         env->vsatp = env->satp;
 150         env->satp = env->satp_hs;
 151     } else {
 152         /* Current V=0 and we are about to change to V=1 */
 153         env->mstatus_hs = env->mstatus & mstatus_mask;
 154         env->mstatus &= ~mstatus_mask;
 155         env->mstatus |= env->vsstatus;
 156
 157 #if defined(TARGET_RISCV32)
 158         env->mstatush_hs = env->mstatush;
 159         env->mstatush |= env->vsstatush;
 160 #endif
 161
 162         env->stvec_hs = env->stvec;
 163         env->stvec = env->vstvec;
 164
 165         env->sscratch_hs = env->sscratch;
 166         env->sscratch = env->vsscratch;
 167
 168         env->sepc_hs = env->sepc;
 169         env->sepc = env->vsepc;
 170
 171         env->scause_hs = env->scause;
 172         env->scause = env->vscause;
 173
 174         env->stval_hs = env->sbadaddr;
 175         env->sbadaddr = env->vstval;
 176
 177         env->satp_hs = env->satp;
 178         env->satp = env->vsatp;
 179     }
 180 }
 181
 182 bool riscv_cpu_virt_enabled(CPURISCVState *env)
 183 {
 184     if (!riscv_has_ext(env, RVH)) {
 185         return false;
 186     }
 187
 188     return get_field(env->virt, VIRT_ONOFF);
 189 }
 190
 191 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable)
 192 {
 193     if (!riscv_has_ext(env, RVH)) {
 194         return;
 195     }
 196
 197     /* Flush the TLB on all virt mode changes. */
 198     if (get_field(env->virt, VIRT_ONOFF) != enable) {
 199         tlb_flush(env_cpu(env));
 200     }
 201
 202     env->virt = set_field(env->virt, VIRT_ONOFF, enable);
 203 }
 204
 205 bool riscv_cpu_force_hs_excep_enabled(CPURISCVState *env)
 206 {
 207     if (!riscv_has_ext(env, RVH)) {
 208         return false;
 209     }
 210
 211     return get_field(env->virt, FORCE_HS_EXCEP);
 212 }
 213
 214 void riscv_cpu_set_force_hs_excep(CPURISCVState *env, bool enable)
 215 {
 216     if (!riscv_has_ext(env, RVH)) {
 217         return;
 218     }
 219
 220     env->virt = set_field(env->virt, FORCE_HS_EXCEP, enable);
 221 }
 222
 223 bool riscv_cpu_two_stage_lookup(CPURISCVState *env)
 224 {
 225     if (!riscv_has_ext(env, RVH)) {
 226         return false;
 227     }
 228
 229     return get_field(env->virt, HS_TWO_STAGE);
 230 }
 231
 232 void riscv_cpu_set_two_stage_lookup(CPURISCVState *env, bool enable)
 233 {
 234     if (!riscv_has_ext(env, RVH)) {
 235         return;
 236     }
 237
 238     env->virt = set_field(env->virt, HS_TWO_STAGE, enable);
 239 }
 240
 241 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint32_t interrupts)
 242 {
 243     CPURISCVState *env = &cpu->env;
 244     if (env->miclaim & interrupts) {
 245         return -1;
 246     } else {
 247         env->miclaim |= interrupts;
 248         return 0;
 249     }
 250 }
 251
 252 uint32_t riscv_cpu_update_mip(RISCVCPU *cpu, uint32_t mask, uint32_t value)
 253 {
 254     CPURISCVState *env = &cpu->env;
 255     CPUState *cs = CPU(cpu);
 256     uint32_t old = env->mip;
 257     bool locked = false;
 258
 259     if (!qemu_mutex_iothread_locked()) {
 260         locked = true;
 261         qemu_mutex_lock_iothread();
 262     }
 263
 264     env->mip = (env->mip & ~mask) | (value & mask);
 265
 266     if (env->mip) {
 267         cpu_interrupt(cs, CPU_INTERRUPT_HARD);
 268     } else {
 269         cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
 270     }
 271
 272     if (locked) {
 273         qemu_mutex_unlock_iothread();
 274     }
 275
 276     return old;
 277 }
 278
 279 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(uint32_t),
 280                              uint32_t arg)
 281 {
 282     env->rdtime_fn = fn;
 283     env->rdtime_fn_arg = arg;
 284 }
 285
 286 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
 287 {
 288     if (newpriv > PRV_M) {
 289         g_assert_not_reached();
 290     }
 291     if (newpriv == PRV_H) {
 292         newpriv = PRV_U;
 293     }
 294     /* tlb_flush is unnecessary as mode is contained in mmu_idx */
 295     env->priv = newpriv;
 296
 297     /*
 298      * Clear the load reservation - otherwise a reservation placed in one
 299      * context/process can be used by another, resulting in an SC succeeding
 300      * incorrectly. Version 2.2 of the ISA specification explicitly requires
 301      * this behaviour, while later revisions say that the kernel "should" use
 302      * an SC instruction to force the yielding of a load reservation on a
 303      * preemptive context switch. As a result, do both.
 304      */
 305     env->load_res = -1;
 306 }
 307
 308 /* get_physical_address - get the physical address for this virtual address
 309  *
 310  * Do a page table walk to obtain the physical address corresponding to a
 311  * virtual address. Returns 0 if the translation was successful
 312  *
 313  * Adapted from Spike's mmu_t::translate and mmu_t::walk
 314  *
 315  * @env: CPURISCVState
 316  * @physical: This will be set to the calculated physical address
 317  * @prot: The returned protection attributes
 318  * @addr: The virtual address to be translated
 319  * @access_type: The type of MMU access
 320  * @mmu_idx: Indicates current privilege level
 321  * @first_stage: Are we in first stage translation?
 322  *               Second stage is used for hypervisor guest translation
 323  * @two_stage: Are we going to perform two stage translation
 324  */
 325 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
 326                                 int *prot, target_ulong addr,
 327                                 int access_type, int mmu_idx,
 328                                 bool first_stage, bool two_stage)
 329 {
 330     /* NOTE: the env->pc value visible here will not be
 331      * correct, but the value visible to the exception handler
 332      * (riscv_cpu_do_interrupt) is correct */
 333     MemTxResult res;
 334     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
 335     int mode = mmu_idx;
 336     bool use_background = false;
 337
 338     /*
 339      * Check if we should use the background registers for the two
 340      * stage translation. We don't need to check if we actually need
 341      * two stage translation as that happened before this function
 342      * was called. Background registers will be used if the guest has
 343      * forced a two stage translation to be on (in HS or M mode).
 344      */
 345     if (riscv_cpu_two_stage_lookup(env) && access_type != MMU_INST_FETCH) {
 346         use_background = true;
 347     }
 348
 349     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 350         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 351             mode = get_field(env->mstatus, MSTATUS_MPP);
 352         }
 353     }
 354
 355     if (first_stage == false) {
 356         /* We are in stage 2 translation, this is similar to stage 1. */
 357         /* Stage 2 is always taken as U-mode */
 358         mode = PRV_U;
 359     }
 360
 361     if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) {
 362         *physical = addr;
 363         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 364         return TRANSLATE_SUCCESS;
 365     }
 366
 367     *prot = 0;
 368
 369     hwaddr base;
 370     int levels, ptidxbits, ptesize, vm, sum, mxr, widened;
 371
 372     if (first_stage == true) {
 373         mxr = get_field(env->mstatus, MSTATUS_MXR);
 374     } else {
 375         mxr = get_field(env->vsstatus, MSTATUS_MXR);
 376     }
 377
 378     if (first_stage == true) {
 379         if (use_background) {
 380             base = (hwaddr)get_field(env->vsatp, SATP_PPN) << PGSHIFT;
 381             vm = get_field(env->vsatp, SATP_MODE);
 382         } else {
 383             base = (hwaddr)get_field(env->satp, SATP_PPN) << PGSHIFT;
 384             vm = get_field(env->satp, SATP_MODE);
 385         }
 386         widened = 0;
 387     } else {
 388         base = (hwaddr)get_field(env->hgatp, HGATP_PPN) << PGSHIFT;
 389         vm = get_field(env->hgatp, HGATP_MODE);
 390         widened = 2;
 391     }
 392     sum = get_field(env->mstatus, MSTATUS_SUM);
 393     switch (vm) {
 394     case VM_1_10_SV32:
 395       levels = 2; ptidxbits = 10; ptesize = 4; break;
 396     case VM_1_10_SV39:
 397       levels = 3; ptidxbits = 9; ptesize = 8; break;
 398     case VM_1_10_SV48:
 399       levels = 4; ptidxbits = 9; ptesize = 8; break;
 400     case VM_1_10_SV57:
 401       levels = 5; ptidxbits = 9; ptesize = 8; break;
 402     case VM_1_10_MBARE:
 403         *physical = addr;
 404         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 405         return TRANSLATE_SUCCESS;
 406     default:
 407       g_assert_not_reached();
 408     }
 409
 410     CPUState *cs = env_cpu(env);
 411     int va_bits = PGSHIFT + levels * ptidxbits + widened;
 412     target_ulong mask, masked_msbs;
 413
 414     if (TARGET_LONG_BITS > (va_bits - 1)) {
 415         mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
 416     } else {
 417         mask = 0;
 418     }
 419     masked_msbs = (addr >> (va_bits - 1)) & mask;
 420
 421     if (masked_msbs != 0 && masked_msbs != mask) {
 422         return TRANSLATE_FAIL;
 423     }
 424
 425     int ptshift = (levels - 1) * ptidxbits;
 426     int i;
 427
 428 #if !TCG_OVERSIZED_GUEST
 429 restart:
 430 #endif
 431     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
 432         target_ulong idx;
 433         if (i == 0) {
 434             idx = (addr >> (PGSHIFT + ptshift)) &
 435                            ((1 << (ptidxbits + widened)) - 1);
 436         } else {
 437             idx = (addr >> (PGSHIFT + ptshift)) &
 438                            ((1 << ptidxbits) - 1);
 439         }
 440
 441         /* check that physical address of PTE is legal */
 442         hwaddr pte_addr;
 443
 444         if (two_stage && first_stage) {
 445             int vbase_prot;
 446             hwaddr vbase;
 447
 448             /* Do the second stage translation on the base PTE address. */
 449             int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
 450                                                  base, MMU_DATA_LOAD,
 451                                                  mmu_idx, false, true);
 452
 453             if (vbase_ret != TRANSLATE_SUCCESS) {
 454                 return vbase_ret;
 455             }
 456
 457             pte_addr = vbase + idx * ptesize;
 458         } else {
 459             pte_addr = base + idx * ptesize;
 460         }
 461
 462         if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 463             !pmp_hart_has_privs(env, pte_addr, sizeof(target_ulong),
 464             1 << MMU_DATA_LOAD, PRV_S)) {
 465             return TRANSLATE_PMP_FAIL;
 466         }
 467
 468 #if defined(TARGET_RISCV32)
 469         target_ulong pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
 470 #elif defined(TARGET_RISCV64)
 471         target_ulong pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
 472 #endif
 473         if (res != MEMTX_OK) {
 474             return TRANSLATE_FAIL;
 475         }
 476
 477         hwaddr ppn = pte >> PTE_PPN_SHIFT;
 478
 479         if (!(pte & PTE_V)) {
 480             /* Invalid PTE */
 481             return TRANSLATE_FAIL;
 482         } else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
 483             /* Inner PTE, continue walking */
 484             base = ppn << PGSHIFT;
 485         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
 486             /* Reserved leaf PTE flags: PTE_W */
 487             return TRANSLATE_FAIL;
 488         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
 489             /* Reserved leaf PTE flags: PTE_W + PTE_X */
 490             return TRANSLATE_FAIL;
 491         } else if ((pte & PTE_U) && ((mode != PRV_U) &&
 492                    (!sum || access_type == MMU_INST_FETCH))) {
 493             /* User PTE flags when not U mode and mstatus.SUM is not set,
 494                or the access type is an instruction fetch */
 495             return TRANSLATE_FAIL;
 496         } else if (!(pte & PTE_U) && (mode != PRV_S)) {
 497             /* Supervisor PTE flags when not S mode */
 498             return TRANSLATE_FAIL;
 499         } else if (ppn & ((1ULL << ptshift) - 1)) {
 500             /* Misaligned PPN */
 501             return TRANSLATE_FAIL;
 502         } else if (access_type == MMU_DATA_LOAD && !((pte & PTE_R) ||
 503                    ((pte & PTE_X) && mxr))) {
 504             /* Read access check failed */
 505             return TRANSLATE_FAIL;
 506         } else if (access_type == MMU_DATA_STORE && !(pte & PTE_W)) {
 507             /* Write access check failed */
 508             return TRANSLATE_FAIL;
 509         } else if (access_type == MMU_INST_FETCH && !(pte & PTE_X)) {
 510             /* Fetch access check failed */
 511             return TRANSLATE_FAIL;
 512         } else {
 513             /* if necessary, set accessed and dirty bits. */
 514             target_ulong updated_pte = pte | PTE_A |
 515                 (access_type == MMU_DATA_STORE ? PTE_D : 0);
 516
 517             /* Page table updates need to be atomic with MTTCG enabled */
 518             if (updated_pte != pte) {
 519                 /*
 520                  * - if accessed or dirty bits need updating, and the PTE is
 521                  *   in RAM, then we do so atomically with a compare and swap.
 522                  * - if the PTE is in IO space or ROM, then it can't be updated
 523                  *   and we return TRANSLATE_FAIL.
 524                  * - if the PTE changed by the time we went to update it, then
 525                  *   it is no longer valid and we must re-walk the page table.
 526                  */
 527                 MemoryRegion *mr;
 528                 hwaddr l = sizeof(target_ulong), addr1;
 529                 mr = address_space_translate(cs->as, pte_addr,
 530                     &addr1, &l, false, MEMTXATTRS_UNSPECIFIED);
 531                 if (memory_region_is_ram(mr)) {
 532                     target_ulong *pte_pa =
 533                         qemu_map_ram_ptr(mr->ram_block, addr1);
 534 #if TCG_OVERSIZED_GUEST
 535                     /* MTTCG is not enabled on oversized TCG guests so
 536                      * page table updates do not need to be atomic */
 537                     *pte_pa = pte = updated_pte;
 538 #else
 539                     target_ulong old_pte =
 540                         qatomic_cmpxchg(pte_pa, pte, updated_pte);
 541                     if (old_pte != pte) {
 542                         goto restart;
 543                     } else {
 544                         pte = updated_pte;
 545                     }
 546 #endif
 547                 } else {
 548                     /* misconfigured PTE in ROM (AD bits are not preset) or
 549                      * PTE is in IO space and can't be updated atomically */
 550                     return TRANSLATE_FAIL;
 551                 }
 552             }
 553
 554             /* for superpage mappings, make a fake leaf PTE for the TLB's
 555                benefit. */
 556             target_ulong vpn = addr >> PGSHIFT;
 557             *physical = ((ppn | (vpn & ((1L << ptshift) - 1))) << PGSHIFT) |
 558                         (addr & ~TARGET_PAGE_MASK);
 559
 560             /* set permissions on the TLB entry */
 561             if ((pte & PTE_R) || ((pte & PTE_X) && mxr)) {
 562                 *prot |= PAGE_READ;
 563             }
 564             if ((pte & PTE_X)) {
 565                 *prot |= PAGE_EXEC;
 566             }
 567             /* add write permission on stores or if the page is already dirty,
 568                so that we TLB miss on later writes to update the dirty bit */
 569             if ((pte & PTE_W) &&
 570                     (access_type == MMU_DATA_STORE || (pte & PTE_D))) {
 571                 *prot |= PAGE_WRITE;
 572             }
 573             return TRANSLATE_SUCCESS;
 574         }
 575     }
 576     return TRANSLATE_FAIL;
 577 }
 578
 579 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
 580                                 MMUAccessType access_type, bool pmp_violation,
 581                                 bool first_stage)
 582 {
 583     CPUState *cs = env_cpu(env);
 584     int page_fault_exceptions;
 585     if (first_stage) {
 586         page_fault_exceptions =
 587             get_field(env->satp, SATP_MODE) != VM_1_10_MBARE &&
 588             !pmp_violation;
 589     } else {
 590         page_fault_exceptions =
 591             get_field(env->hgatp, HGATP_MODE) != VM_1_10_MBARE &&
 592             !pmp_violation;
 593     }
 594     switch (access_type) {
 595     case MMU_INST_FETCH:
 596         if (riscv_cpu_virt_enabled(env) && !first_stage) {
 597             cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
 598         } else {
 599             cs->exception_index = page_fault_exceptions ?
 600                 RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
 601         }
 602         break;
 603     case MMU_DATA_LOAD:
 604         if ((riscv_cpu_virt_enabled(env) || riscv_cpu_two_stage_lookup(env)) &&
 605             !first_stage) {
 606             cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
 607         } else {
 608             cs->exception_index = page_fault_exceptions ?
 609                 RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
 610         }
 611         break;
 612     case MMU_DATA_STORE:
 613         if ((riscv_cpu_virt_enabled(env) || riscv_cpu_two_stage_lookup(env)) &&
 614             !first_stage) {
 615             cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
 616         } else {
 617             cs->exception_index = page_fault_exceptions ?
 618                 RISCV_EXCP_STORE_PAGE_FAULT : RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 619         }
 620         break;
 621     default:
 622         g_assert_not_reached();
 623     }
 624     env->badaddr = address;
 625 }
 626
 627 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
 628 {
 629     RISCVCPU *cpu = RISCV_CPU(cs);
 630     CPURISCVState *env = &cpu->env;
 631     hwaddr phys_addr;
 632     int prot;
 633     int mmu_idx = cpu_mmu_index(&cpu->env, false);
 634
 635     if (get_physical_address(env, &phys_addr, &prot, addr, 0, mmu_idx,
 636                              true, riscv_cpu_virt_enabled(env))) {
 637         return -1;
 638     }
 639
 640     if (riscv_cpu_virt_enabled(env)) {
 641         if (get_physical_address(env, &phys_addr, &prot, phys_addr,
 642                                  0, mmu_idx, false, true)) {
 643             return -1;
 644         }
 645     }
 646
 647     return phys_addr & TARGET_PAGE_MASK;
 648 }
 649
 650 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
 651                                      vaddr addr, unsigned size,
 652                                      MMUAccessType access_type,
 653                                      int mmu_idx, MemTxAttrs attrs,
 654                                      MemTxResult response, uintptr_t retaddr)
 655 {
 656     RISCVCPU *cpu = RISCV_CPU(cs);
 657     CPURISCVState *env = &cpu->env;
 658
 659     if (access_type == MMU_DATA_STORE) {
 660         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 661     } else {
 662         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
 663     }
 664
 665     env->badaddr = addr;
 666     riscv_raise_exception(&cpu->env, cs->exception_index, retaddr);
 667 }
 668
 669 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
 670                                    MMUAccessType access_type, int mmu_idx,
 671                                    uintptr_t retaddr)
 672 {
 673     RISCVCPU *cpu = RISCV_CPU(cs);
 674     CPURISCVState *env = &cpu->env;
 675     switch (access_type) {
 676     case MMU_INST_FETCH:
 677         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
 678         break;
 679     case MMU_DATA_LOAD:
 680         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
 681         break;
 682     case MMU_DATA_STORE:
 683         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
 684         break;
 685     default:
 686         g_assert_not_reached();
 687     }
 688     env->badaddr = addr;
 689     riscv_raise_exception(env, cs->exception_index, retaddr);
 690 }
 691 #endif
 692
 693 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
 694                         MMUAccessType access_type, int mmu_idx,
 695                         bool probe, uintptr_t retaddr)
 696 {
 697     RISCVCPU *cpu = RISCV_CPU(cs);
 698     CPURISCVState *env = &cpu->env;
 699 #ifndef CONFIG_USER_ONLY
 700     vaddr im_address;
 701     hwaddr pa = 0;
 702     int prot, prot2;
 703     bool pmp_violation = false;
 704     bool first_stage_error = true;
 705     int ret = TRANSLATE_FAIL;
 706     int mode = mmu_idx;
 707     target_ulong tlb_size = 0;
 708
 709     env->guest_phys_fault_addr = 0;
 710
 711     qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
 712                   __func__, address, access_type, mmu_idx);
 713
 714     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 715         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 716             mode = get_field(env->mstatus, MSTATUS_MPP);
 717         }
 718     }
 719
 720     if (riscv_has_ext(env, RVH) && env->priv == PRV_M &&
 721         access_type != MMU_INST_FETCH &&
 722         get_field(env->mstatus, MSTATUS_MPRV) &&
 723         MSTATUS_MPV_ISSET(env)) {
 724         riscv_cpu_set_two_stage_lookup(env, true);
 725     }
 726
 727     if (riscv_cpu_virt_enabled(env) ||
 728         (riscv_cpu_two_stage_lookup(env) && access_type != MMU_INST_FETCH)) {
 729         /* Two stage lookup */
 730         ret = get_physical_address(env, &pa, &prot, address, access_type,
 731                                    mmu_idx, true, true);
 732
 733         qemu_log_mask(CPU_LOG_MMU,
 734                       "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
 735                       TARGET_FMT_plx " prot %d\n",
 736                       __func__, address, ret, pa, prot);
 737
 738         if (ret != TRANSLATE_FAIL) {
 739             /* Second stage lookup */
 740             im_address = pa;
 741
 742             ret = get_physical_address(env, &pa, &prot2, im_address,
 743                                        access_type, mmu_idx, false, true);
 744
 745             qemu_log_mask(CPU_LOG_MMU,
 746                     "%s 2nd-stage address=%" VADDR_PRIx " ret %d physical "
 747                     TARGET_FMT_plx " prot %d\n",
 748                     __func__, im_address, ret, pa, prot2);
 749
 750             prot &= prot2;
 751
 752             if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 753                 (ret == TRANSLATE_SUCCESS) &&
 754                 !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
 755                 ret = TRANSLATE_PMP_FAIL;
 756             }
 757
 758             if (ret != TRANSLATE_SUCCESS) {
 759                 /*
 760                  * Guest physical address translation failed, this is a HS
 761                  * level exception
 762                  */
 763                 first_stage_error = false;
 764                 env->guest_phys_fault_addr = (im_address |
 765                                               (address &
 766                                                (TARGET_PAGE_SIZE - 1))) >> 2;
 767             }
 768         }
 769     } else {
 770         /* Single stage lookup */
 771         ret = get_physical_address(env, &pa, &prot, address, access_type,
 772                                    mmu_idx, true, false);
 773
 774         qemu_log_mask(CPU_LOG_MMU,
 775                       "%s address=%" VADDR_PRIx " ret %d physical "
 776                       TARGET_FMT_plx " prot %d\n",
 777                       __func__, address, ret, pa, prot);
 778     }
 779
 780     /* We did the two stage lookup based on MPRV, unset the lookup */
 781     if (riscv_has_ext(env, RVH) && env->priv == PRV_M &&
 782         access_type != MMU_INST_FETCH &&
 783         get_field(env->mstatus, MSTATUS_MPRV) &&
 784         MSTATUS_MPV_ISSET(env)) {
 785         riscv_cpu_set_two_stage_lookup(env, false);
 786     }
 787
 788     if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 789         (ret == TRANSLATE_SUCCESS) &&
 790         !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
 791         ret = TRANSLATE_PMP_FAIL;
 792     }
 793     if (ret == TRANSLATE_PMP_FAIL) {
 794         pmp_violation = true;
 795     }
 796
 797     if (ret == TRANSLATE_SUCCESS) {
 798         if (pmp_is_range_in_tlb(env, pa & TARGET_PAGE_MASK, &tlb_size)) {
 799             tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
 800                          prot, mmu_idx, tlb_size);
 801         } else {
 802             tlb_set_page(cs, address & TARGET_PAGE_MASK, pa & TARGET_PAGE_MASK,
 803                          prot, mmu_idx, TARGET_PAGE_SIZE);
 804         }
 805         return true;
 806     } else if (probe) {
 807         return false;
 808     } else {
 809         raise_mmu_exception(env, address, access_type, pmp_violation, first_stage_error);
 810         riscv_raise_exception(env, cs->exception_index, retaddr);
 811     }
 812
 813     return true;
 814
 815 #else
 816     switch (access_type) {
 817     case MMU_INST_FETCH:
 818         cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
 819         break;
 820     case MMU_DATA_LOAD:
 821         cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
 822         break;
 823     case MMU_DATA_STORE:
 824         cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
 825         break;
 826     default:
 827         g_assert_not_reached();
 828     }
 829     env->badaddr = address;
 830     cpu_loop_exit_restore(cs, retaddr);
 831 #endif
 832 }
 833
 834 /*
 835  * Handle Traps
 836  *
 837  * Adapted from Spike's processor_t::take_trap.
 838  *
 839  */
 840 void riscv_cpu_do_interrupt(CPUState *cs)
 841 {
 842 #if !defined(CONFIG_USER_ONLY)
 843
 844     RISCVCPU *cpu = RISCV_CPU(cs);
 845     CPURISCVState *env = &cpu->env;
 846     bool force_hs_execp = riscv_cpu_force_hs_excep_enabled(env);
 847     target_ulong s;
 848
 849     /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
 850      * so we mask off the MSB and separate into trap type and cause.
 851      */
 852     bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
 853     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
 854     target_ulong deleg = async ? env->mideleg : env->medeleg;
 855     target_ulong tval = 0;
 856     target_ulong htval = 0;
 857     target_ulong mtval2 = 0;
 858
 859     if (!async) {
 860         /* set tval to badaddr for traps with address information */
 861         switch (cause) {
 862         case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
 863         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
 864         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
 865             force_hs_execp = true;
 866             /* fallthrough */
 867         case RISCV_EXCP_INST_ADDR_MIS:
 868         case RISCV_EXCP_INST_ACCESS_FAULT:
 869         case RISCV_EXCP_LOAD_ADDR_MIS:
 870         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
 871         case RISCV_EXCP_LOAD_ACCESS_FAULT:
 872         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
 873         case RISCV_EXCP_INST_PAGE_FAULT:
 874         case RISCV_EXCP_LOAD_PAGE_FAULT:
 875         case RISCV_EXCP_STORE_PAGE_FAULT:
 876             tval = env->badaddr;
 877             break;
 878         default:
 879             break;
 880         }
 881         /* ecall is dispatched as one cause so translate based on mode */
 882         if (cause == RISCV_EXCP_U_ECALL) {
 883             assert(env->priv <= 3);
 884
 885             if (env->priv == PRV_M) {
 886                 cause = RISCV_EXCP_M_ECALL;
 887             } else if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
 888                 cause = RISCV_EXCP_VS_ECALL;
 889             } else if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) {
 890                 cause = RISCV_EXCP_S_ECALL;
 891             } else if (env->priv == PRV_U) {
 892                 cause = RISCV_EXCP_U_ECALL;
 893             }
 894         }
 895     }
 896
 897     trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
 898         riscv_cpu_get_trap_name(cause, async));
 899
 900     if (env->priv <= PRV_S &&
 901             cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
 902         /* handle the trap in S-mode */
 903         if (riscv_has_ext(env, RVH)) {
 904             target_ulong hdeleg = async ? env->hideleg : env->hedeleg;
 905
 906             if ((riscv_cpu_virt_enabled(env) ||
 907                  riscv_cpu_two_stage_lookup(env)) && tval) {
 908                 /*
 909                  * If we are writing a guest virtual address to stval, set
 910                  * this to 1. If we are trapping to VS we will set this to 0
 911                  * later.
 912                  */
 913                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 1);
 914             } else {
 915                 /* For other HS-mode traps, we set this to 0. */
 916                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
 917             }
 918
 919             if (riscv_cpu_virt_enabled(env) && ((hdeleg >> cause) & 1) &&
 920                 !force_hs_execp) {
 921                 /* Trap to VS mode */
 922                 /*
 923                  * See if we need to adjust cause. Yes if its VS mode interrupt
 924                  * no if hypervisor has delegated one of hs mode's interrupt
 925                  */
 926                 if (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
 927                     cause == IRQ_VS_EXT) {
 928                     cause = cause - 1;
 929                 }
 930                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
 931             } else if (riscv_cpu_virt_enabled(env)) {
 932                 /* Trap into HS mode, from virt */
 933                 riscv_cpu_swap_hypervisor_regs(env);
 934                 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
 935                                          get_field(env->mstatus, SSTATUS_SPP));
 936                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 937                                          riscv_cpu_virt_enabled(env));
 938
 939                 htval = env->guest_phys_fault_addr;
 940
 941                 riscv_cpu_set_virt_enabled(env, 0);
 942                 riscv_cpu_set_force_hs_excep(env, 0);
 943             } else {
 944                 /* Trap into HS mode */
 945                 if (!riscv_cpu_two_stage_lookup(env)) {
 946                     env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 947                                              riscv_cpu_virt_enabled(env));
 948                 }
 949                 riscv_cpu_set_two_stage_lookup(env, false);
 950                 htval = env->guest_phys_fault_addr;
 951             }
 952         }
 953
 954         s = env->mstatus;
 955         s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
 956         s = set_field(s, MSTATUS_SPP, env->priv);
 957         s = set_field(s, MSTATUS_SIE, 0);
 958         env->mstatus = s;
 959         env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
 960         env->sepc = env->pc;
 961         env->sbadaddr = tval;
 962         env->htval = htval;
 963         env->pc = (env->stvec >> 2 << 2) +
 964             ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
 965         riscv_cpu_set_mode(env, PRV_S);
 966     } else {
 967         /* handle the trap in M-mode */
 968         if (riscv_has_ext(env, RVH)) {
 969             if (riscv_cpu_virt_enabled(env)) {
 970                 riscv_cpu_swap_hypervisor_regs(env);
 971             }
 972 #ifdef TARGET_RISCV32
 973             env->mstatush = set_field(env->mstatush, MSTATUS_MPV,
 974                                        riscv_cpu_virt_enabled(env));
 975             if (riscv_cpu_virt_enabled(env) && tval) {
 976                 env->mstatush = set_field(env->mstatush, MSTATUS_GVA, 1);
 977             }
 978 #else
 979             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
 980                                       riscv_cpu_virt_enabled(env));
 981             if (riscv_cpu_virt_enabled(env) && tval) {
 982                 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
 983             }
 984 #endif
 985
 986             mtval2 = env->guest_phys_fault_addr;
 987
 988             /* Trapping to M mode, virt is disabled */
 989             riscv_cpu_set_virt_enabled(env, 0);
 990             riscv_cpu_set_force_hs_excep(env, 0);
 991         }
 992
 993         s = env->mstatus;
 994         s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
 995         s = set_field(s, MSTATUS_MPP, env->priv);
 996         s = set_field(s, MSTATUS_MIE, 0);
 997         env->mstatus = s;
 998         env->mcause = cause | ~(((target_ulong)-1) >> async);
 999         env->mepc = env->pc;
1000         env->mbadaddr = tval;
1001         env->mtval2 = mtval2;
1002         env->pc = (env->mtvec >> 2 << 2) +
1003             ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
1004         riscv_cpu_set_mode(env, PRV_M);
1005     }
1006
1007     /* NOTE: it is not necessary to yield load reservations here. It is only
1008      * necessary for an SC from "another hart" to cause a load reservation
1009      * to be yielded. Refer to the memory consistency model section of the
1010      * RISC-V ISA Specification.
1011      */
1012
1013 #endif
1014     cs->exception_index = EXCP_NONE; /* mark handled to qemu */
1015 }