target/riscv/cpu_helper.c

   1 /*
   2  * RISC-V CPU helpers for qemu.
   3  *
   4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
   5  * Copyright (c) 2017-2018 SiFive, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2 or later, as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along with
  17  * this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "qemu/main-loop.h"
  23 #include "cpu.h"
  24 #include "exec/exec-all.h"
  25 #include "tcg/tcg-op.h"
  26 #include "trace.h"
  27 #include "hw/semihosting/common-semi.h"
  28
  29 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
  30 {
  31 #ifdef CONFIG_USER_ONLY
  32     return 0;
  33 #else
  34     return env->priv;
  35 #endif
  36 }
  37
  38 #ifndef CONFIG_USER_ONLY
  39 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
  40 {
  41     target_ulong irqs;
  42
  43     target_ulong mstatus_mie = get_field(env->mstatus, MSTATUS_MIE);
  44     target_ulong mstatus_sie = get_field(env->mstatus, MSTATUS_SIE);
  45     target_ulong hs_mstatus_sie = get_field(env->mstatus_hs, MSTATUS_SIE);
  46
  47     target_ulong pending = env->mip & env->mie &
  48                                ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
  49     target_ulong vspending = (env->mip & env->mie &
  50                               (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP));
  51
  52     target_ulong mie    = env->priv < PRV_M ||
  53                           (env->priv == PRV_M && mstatus_mie);
  54     target_ulong sie    = env->priv < PRV_S ||
  55                           (env->priv == PRV_S && mstatus_sie);
  56     target_ulong hs_sie = env->priv < PRV_S ||
  57                           (env->priv == PRV_S && hs_mstatus_sie);
  58
  59     if (riscv_cpu_virt_enabled(env)) {
  60         target_ulong pending_hs_irq = pending & -hs_sie;
  61
  62         if (pending_hs_irq) {
  63             riscv_cpu_set_force_hs_excep(env, FORCE_HS_EXCEP);
  64             return ctz64(pending_hs_irq);
  65         }
  66
  67         pending = vspending;
  68     }
  69
  70     irqs = (pending & ~env->mideleg & -mie) | (pending &  env->mideleg & -sie);
  71
  72     if (irqs) {
  73         return ctz64(irqs); /* since non-zero */
  74     } else {
  75         return EXCP_NONE; /* indicates no pending interrupt */
  76     }
  77 }
  78 #endif
  79
  80 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
  81 {
  82 #if !defined(CONFIG_USER_ONLY)
  83     if (interrupt_request & CPU_INTERRUPT_HARD) {
  84         RISCVCPU *cpu = RISCV_CPU(cs);
  85         CPURISCVState *env = &cpu->env;
  86         int interruptno = riscv_cpu_local_irq_pending(env);
  87         if (interruptno >= 0) {
  88             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
  89             riscv_cpu_do_interrupt(cs);
  90             return true;
  91         }
  92     }
  93 #endif
  94     return false;
  95 }
  96
  97 #if !defined(CONFIG_USER_ONLY)
  98
  99 /* Return true is floating point support is currently enabled */
 100 bool riscv_cpu_fp_enabled(CPURISCVState *env)
 101 {
 102     if (env->mstatus & MSTATUS_FS) {
 103         if (riscv_cpu_virt_enabled(env) && !(env->mstatus_hs & MSTATUS_FS)) {
 104             return false;
 105         }
 106         return true;
 107     }
 108
 109     return false;
 110 }
 111
 112 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
 113 {
 114     uint64_t mstatus_mask = MSTATUS_MXR | MSTATUS_SUM | MSTATUS_FS |
 115                             MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE |
 116                             MSTATUS64_UXL;
 117     bool current_virt = riscv_cpu_virt_enabled(env);
 118
 119     g_assert(riscv_has_ext(env, RVH));
 120
 121     if (current_virt) {
 122         /* Current V=1 and we are about to change to V=0 */
 123         env->vsstatus = env->mstatus & mstatus_mask;
 124         env->mstatus &= ~mstatus_mask;
 125         env->mstatus |= env->mstatus_hs;
 126
 127         env->vstvec = env->stvec;
 128         env->stvec = env->stvec_hs;
 129
 130         env->vsscratch = env->sscratch;
 131         env->sscratch = env->sscratch_hs;
 132
 133         env->vsepc = env->sepc;
 134         env->sepc = env->sepc_hs;
 135
 136         env->vscause = env->scause;
 137         env->scause = env->scause_hs;
 138
 139         env->vstval = env->sbadaddr;
 140         env->sbadaddr = env->stval_hs;
 141
 142         env->vsatp = env->satp;
 143         env->satp = env->satp_hs;
 144     } else {
 145         /* Current V=0 and we are about to change to V=1 */
 146         env->mstatus_hs = env->mstatus & mstatus_mask;
 147         env->mstatus &= ~mstatus_mask;
 148         env->mstatus |= env->vsstatus;
 149
 150         env->stvec_hs = env->stvec;
 151         env->stvec = env->vstvec;
 152
 153         env->sscratch_hs = env->sscratch;
 154         env->sscratch = env->vsscratch;
 155
 156         env->sepc_hs = env->sepc;
 157         env->sepc = env->vsepc;
 158
 159         env->scause_hs = env->scause;
 160         env->scause = env->vscause;
 161
 162         env->stval_hs = env->sbadaddr;
 163         env->sbadaddr = env->vstval;
 164
 165         env->satp_hs = env->satp;
 166         env->satp = env->vsatp;
 167     }
 168 }
 169
 170 bool riscv_cpu_virt_enabled(CPURISCVState *env)
 171 {
 172     if (!riscv_has_ext(env, RVH)) {
 173         return false;
 174     }
 175
 176     return get_field(env->virt, VIRT_ONOFF);
 177 }
 178
 179 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable)
 180 {
 181     if (!riscv_has_ext(env, RVH)) {
 182         return;
 183     }
 184
 185     /* Flush the TLB on all virt mode changes. */
 186     if (get_field(env->virt, VIRT_ONOFF) != enable) {
 187         tlb_flush(env_cpu(env));
 188     }
 189
 190     env->virt = set_field(env->virt, VIRT_ONOFF, enable);
 191 }
 192
 193 bool riscv_cpu_force_hs_excep_enabled(CPURISCVState *env)
 194 {
 195     if (!riscv_has_ext(env, RVH)) {
 196         return false;
 197     }
 198
 199     return get_field(env->virt, FORCE_HS_EXCEP);
 200 }
 201
 202 void riscv_cpu_set_force_hs_excep(CPURISCVState *env, bool enable)
 203 {
 204     if (!riscv_has_ext(env, RVH)) {
 205         return;
 206     }
 207
 208     env->virt = set_field(env->virt, FORCE_HS_EXCEP, enable);
 209 }
 210
 211 bool riscv_cpu_two_stage_lookup(int mmu_idx)
 212 {
 213     return mmu_idx & TB_FLAGS_PRIV_HYP_ACCESS_MASK;
 214 }
 215
 216 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint32_t interrupts)
 217 {
 218     CPURISCVState *env = &cpu->env;
 219     if (env->miclaim & interrupts) {
 220         return -1;
 221     } else {
 222         env->miclaim |= interrupts;
 223         return 0;
 224     }
 225 }
 226
 227 uint32_t riscv_cpu_update_mip(RISCVCPU *cpu, uint32_t mask, uint32_t value)
 228 {
 229     CPURISCVState *env = &cpu->env;
 230     CPUState *cs = CPU(cpu);
 231     uint32_t old = env->mip;
 232     bool locked = false;
 233
 234     if (!qemu_mutex_iothread_locked()) {
 235         locked = true;
 236         qemu_mutex_lock_iothread();
 237     }
 238
 239     env->mip = (env->mip & ~mask) | (value & mask);
 240
 241     if (env->mip) {
 242         cpu_interrupt(cs, CPU_INTERRUPT_HARD);
 243     } else {
 244         cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
 245     }
 246
 247     if (locked) {
 248         qemu_mutex_unlock_iothread();
 249     }
 250
 251     return old;
 252 }
 253
 254 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(uint32_t),
 255                              uint32_t arg)
 256 {
 257     env->rdtime_fn = fn;
 258     env->rdtime_fn_arg = arg;
 259 }
 260
 261 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
 262 {
 263     if (newpriv > PRV_M) {
 264         g_assert_not_reached();
 265     }
 266     if (newpriv == PRV_H) {
 267         newpriv = PRV_U;
 268     }
 269     /* tlb_flush is unnecessary as mode is contained in mmu_idx */
 270     env->priv = newpriv;
 271
 272     /*
 273      * Clear the load reservation - otherwise a reservation placed in one
 274      * context/process can be used by another, resulting in an SC succeeding
 275      * incorrectly. Version 2.2 of the ISA specification explicitly requires
 276      * this behaviour, while later revisions say that the kernel "should" use
 277      * an SC instruction to force the yielding of a load reservation on a
 278      * preemptive context switch. As a result, do both.
 279      */
 280     env->load_res = -1;
 281 }
 282
 283 /* get_physical_address - get the physical address for this virtual address
 284  *
 285  * Do a page table walk to obtain the physical address corresponding to a
 286  * virtual address. Returns 0 if the translation was successful
 287  *
 288  * Adapted from Spike's mmu_t::translate and mmu_t::walk
 289  *
 290  * @env: CPURISCVState
 291  * @physical: This will be set to the calculated physical address
 292  * @prot: The returned protection attributes
 293  * @addr: The virtual address to be translated
 294  * @fault_pte_addr: If not NULL, this will be set to fault pte address
 295  *                  when a error occurs on pte address translation.
 296  *                  This will already be shifted to match htval.
 297  * @access_type: The type of MMU access
 298  * @mmu_idx: Indicates current privilege level
 299  * @first_stage: Are we in first stage translation?
 300  *               Second stage is used for hypervisor guest translation
 301  * @two_stage: Are we going to perform two stage translation
 302  */
 303 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
 304                                 int *prot, target_ulong addr,
 305                                 target_ulong *fault_pte_addr,
 306                                 int access_type, int mmu_idx,
 307                                 bool first_stage, bool two_stage)
 308 {
 309     /* NOTE: the env->pc value visible here will not be
 310      * correct, but the value visible to the exception handler
 311      * (riscv_cpu_do_interrupt) is correct */
 312     MemTxResult res;
 313     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
 314     int mode = mmu_idx & TB_FLAGS_PRIV_MMU_MASK;
 315     bool use_background = false;
 316
 317     /*
 318      * Check if we should use the background registers for the two
 319      * stage translation. We don't need to check if we actually need
 320      * two stage translation as that happened before this function
 321      * was called. Background registers will be used if the guest has
 322      * forced a two stage translation to be on (in HS or M mode).
 323      */
 324     if (!riscv_cpu_virt_enabled(env) && riscv_cpu_two_stage_lookup(mmu_idx)) {
 325         use_background = true;
 326     }
 327
 328     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 329         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 330             mode = get_field(env->mstatus, MSTATUS_MPP);
 331         }
 332     }
 333
 334     if (first_stage == false) {
 335         /* We are in stage 2 translation, this is similar to stage 1. */
 336         /* Stage 2 is always taken as U-mode */
 337         mode = PRV_U;
 338     }
 339
 340     if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) {
 341         *physical = addr;
 342         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 343         return TRANSLATE_SUCCESS;
 344     }
 345
 346     *prot = 0;
 347
 348     hwaddr base;
 349     int levels, ptidxbits, ptesize, vm, sum, mxr, widened;
 350
 351     if (first_stage == true) {
 352         mxr = get_field(env->mstatus, MSTATUS_MXR);
 353     } else {
 354         mxr = get_field(env->vsstatus, MSTATUS_MXR);
 355     }
 356
 357     if (first_stage == true) {
 358         if (use_background) {
 359             base = (hwaddr)get_field(env->vsatp, SATP_PPN) << PGSHIFT;
 360             vm = get_field(env->vsatp, SATP_MODE);
 361         } else {
 362             base = (hwaddr)get_field(env->satp, SATP_PPN) << PGSHIFT;
 363             vm = get_field(env->satp, SATP_MODE);
 364         }
 365         widened = 0;
 366     } else {
 367         base = (hwaddr)get_field(env->hgatp, HGATP_PPN) << PGSHIFT;
 368         vm = get_field(env->hgatp, HGATP_MODE);
 369         widened = 2;
 370     }
 371     /* status.SUM will be ignored if execute on background */
 372     sum = get_field(env->mstatus, MSTATUS_SUM) || use_background;
 373     switch (vm) {
 374     case VM_1_10_SV32:
 375       levels = 2; ptidxbits = 10; ptesize = 4; break;
 376     case VM_1_10_SV39:
 377       levels = 3; ptidxbits = 9; ptesize = 8; break;
 378     case VM_1_10_SV48:
 379       levels = 4; ptidxbits = 9; ptesize = 8; break;
 380     case VM_1_10_SV57:
 381       levels = 5; ptidxbits = 9; ptesize = 8; break;
 382     case VM_1_10_MBARE:
 383         *physical = addr;
 384         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 385         return TRANSLATE_SUCCESS;
 386     default:
 387       g_assert_not_reached();
 388     }
 389
 390     CPUState *cs = env_cpu(env);
 391     int va_bits = PGSHIFT + levels * ptidxbits + widened;
 392     target_ulong mask, masked_msbs;
 393
 394     if (TARGET_LONG_BITS > (va_bits - 1)) {
 395         mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
 396     } else {
 397         mask = 0;
 398     }
 399     masked_msbs = (addr >> (va_bits - 1)) & mask;
 400
 401     if (masked_msbs != 0 && masked_msbs != mask) {
 402         return TRANSLATE_FAIL;
 403     }
 404
 405     int ptshift = (levels - 1) * ptidxbits;
 406     int i;
 407
 408 #if !TCG_OVERSIZED_GUEST
 409 restart:
 410 #endif
 411     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
 412         target_ulong idx;
 413         if (i == 0) {
 414             idx = (addr >> (PGSHIFT + ptshift)) &
 415                            ((1 << (ptidxbits + widened)) - 1);
 416         } else {
 417             idx = (addr >> (PGSHIFT + ptshift)) &
 418                            ((1 << ptidxbits) - 1);
 419         }
 420
 421         /* check that physical address of PTE is legal */
 422         hwaddr pte_addr;
 423
 424         if (two_stage && first_stage) {
 425             int vbase_prot;
 426             hwaddr vbase;
 427
 428             /* Do the second stage translation on the base PTE address. */
 429             int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
 430                                                  base, NULL, MMU_DATA_LOAD,
 431                                                  mmu_idx, false, true);
 432
 433             if (vbase_ret != TRANSLATE_SUCCESS) {
 434                 if (fault_pte_addr) {
 435                     *fault_pte_addr = (base + idx * ptesize) >> 2;
 436                 }
 437                 return TRANSLATE_G_STAGE_FAIL;
 438             }
 439
 440             pte_addr = vbase + idx * ptesize;
 441         } else {
 442             pte_addr = base + idx * ptesize;
 443         }
 444
 445         if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 446             !pmp_hart_has_privs(env, pte_addr, sizeof(target_ulong),
 447             1 << MMU_DATA_LOAD, PRV_S)) {
 448             return TRANSLATE_PMP_FAIL;
 449         }
 450
 451         target_ulong pte;
 452         if (riscv_cpu_is_32bit(env)) {
 453             pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
 454         } else {
 455             pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
 456         }
 457
 458         if (res != MEMTX_OK) {
 459             return TRANSLATE_FAIL;
 460         }
 461
 462         hwaddr ppn = pte >> PTE_PPN_SHIFT;
 463
 464         if (!(pte & PTE_V)) {
 465             /* Invalid PTE */
 466             return TRANSLATE_FAIL;
 467         } else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
 468             /* Inner PTE, continue walking */
 469             base = ppn << PGSHIFT;
 470         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
 471             /* Reserved leaf PTE flags: PTE_W */
 472             return TRANSLATE_FAIL;
 473         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
 474             /* Reserved leaf PTE flags: PTE_W + PTE_X */
 475             return TRANSLATE_FAIL;
 476         } else if ((pte & PTE_U) && ((mode != PRV_U) &&
 477                    (!sum || access_type == MMU_INST_FETCH))) {
 478             /* User PTE flags when not U mode and mstatus.SUM is not set,
 479                or the access type is an instruction fetch */
 480             return TRANSLATE_FAIL;
 481         } else if (!(pte & PTE_U) && (mode != PRV_S)) {
 482             /* Supervisor PTE flags when not S mode */
 483             return TRANSLATE_FAIL;
 484         } else if (ppn & ((1ULL << ptshift) - 1)) {
 485             /* Misaligned PPN */
 486             return TRANSLATE_FAIL;
 487         } else if (access_type == MMU_DATA_LOAD && !((pte & PTE_R) ||
 488                    ((pte & PTE_X) && mxr))) {
 489             /* Read access check failed */
 490             return TRANSLATE_FAIL;
 491         } else if (access_type == MMU_DATA_STORE && !(pte & PTE_W)) {
 492             /* Write access check failed */
 493             return TRANSLATE_FAIL;
 494         } else if (access_type == MMU_INST_FETCH && !(pte & PTE_X)) {
 495             /* Fetch access check failed */
 496             return TRANSLATE_FAIL;
 497         } else {
 498             /* if necessary, set accessed and dirty bits. */
 499             target_ulong updated_pte = pte | PTE_A |
 500                 (access_type == MMU_DATA_STORE ? PTE_D : 0);
 501
 502             /* Page table updates need to be atomic with MTTCG enabled */
 503             if (updated_pte != pte) {
 504                 /*
 505                  * - if accessed or dirty bits need updating, and the PTE is
 506                  *   in RAM, then we do so atomically with a compare and swap.
 507                  * - if the PTE is in IO space or ROM, then it can't be updated
 508                  *   and we return TRANSLATE_FAIL.
 509                  * - if the PTE changed by the time we went to update it, then
 510                  *   it is no longer valid and we must re-walk the page table.
 511                  */
 512                 MemoryRegion *mr;
 513                 hwaddr l = sizeof(target_ulong), addr1;
 514                 mr = address_space_translate(cs->as, pte_addr,
 515                     &addr1, &l, false, MEMTXATTRS_UNSPECIFIED);
 516                 if (memory_region_is_ram(mr)) {
 517                     target_ulong *pte_pa =
 518                         qemu_map_ram_ptr(mr->ram_block, addr1);
 519 #if TCG_OVERSIZED_GUEST
 520                     /* MTTCG is not enabled on oversized TCG guests so
 521                      * page table updates do not need to be atomic */
 522                     *pte_pa = pte = updated_pte;
 523 #else
 524                     target_ulong old_pte =
 525                         qatomic_cmpxchg(pte_pa, pte, updated_pte);
 526                     if (old_pte != pte) {
 527                         goto restart;
 528                     } else {
 529                         pte = updated_pte;
 530                     }
 531 #endif
 532                 } else {
 533                     /* misconfigured PTE in ROM (AD bits are not preset) or
 534                      * PTE is in IO space and can't be updated atomically */
 535                     return TRANSLATE_FAIL;
 536                 }
 537             }
 538
 539             /* for superpage mappings, make a fake leaf PTE for the TLB's
 540                benefit. */
 541             target_ulong vpn = addr >> PGSHIFT;
 542             *physical = ((ppn | (vpn & ((1L << ptshift) - 1))) << PGSHIFT) |
 543                         (addr & ~TARGET_PAGE_MASK);
 544
 545             /* set permissions on the TLB entry */
 546             if ((pte & PTE_R) || ((pte & PTE_X) && mxr)) {
 547                 *prot |= PAGE_READ;
 548             }
 549             if ((pte & PTE_X)) {
 550                 *prot |= PAGE_EXEC;
 551             }
 552             /* add write permission on stores or if the page is already dirty,
 553                so that we TLB miss on later writes to update the dirty bit */
 554             if ((pte & PTE_W) &&
 555                     (access_type == MMU_DATA_STORE || (pte & PTE_D))) {
 556                 *prot |= PAGE_WRITE;
 557             }
 558             return TRANSLATE_SUCCESS;
 559         }
 560     }
 561     return TRANSLATE_FAIL;
 562 }
 563
 564 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
 565                                 MMUAccessType access_type, bool pmp_violation,
 566                                 bool first_stage, bool two_stage)
 567 {
 568     CPUState *cs = env_cpu(env);
 569     int page_fault_exceptions;
 570     if (first_stage) {
 571         page_fault_exceptions =
 572             get_field(env->satp, SATP_MODE) != VM_1_10_MBARE &&
 573             !pmp_violation;
 574     } else {
 575         page_fault_exceptions =
 576             get_field(env->hgatp, HGATP_MODE) != VM_1_10_MBARE &&
 577             !pmp_violation;
 578     }
 579     switch (access_type) {
 580     case MMU_INST_FETCH:
 581         if (riscv_cpu_virt_enabled(env) && !first_stage) {
 582             cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
 583         } else {
 584             cs->exception_index = page_fault_exceptions ?
 585                 RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
 586         }
 587         break;
 588     case MMU_DATA_LOAD:
 589         if (two_stage && !first_stage) {
 590             cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
 591         } else {
 592             cs->exception_index = page_fault_exceptions ?
 593                 RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
 594         }
 595         break;
 596     case MMU_DATA_STORE:
 597         if (two_stage && !first_stage) {
 598             cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
 599         } else {
 600             cs->exception_index = page_fault_exceptions ?
 601                 RISCV_EXCP_STORE_PAGE_FAULT : RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 602         }
 603         break;
 604     default:
 605         g_assert_not_reached();
 606     }
 607     env->badaddr = address;
 608 }
 609
 610 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
 611 {
 612     RISCVCPU *cpu = RISCV_CPU(cs);
 613     CPURISCVState *env = &cpu->env;
 614     hwaddr phys_addr;
 615     int prot;
 616     int mmu_idx = cpu_mmu_index(&cpu->env, false);
 617
 618     if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
 619                              true, riscv_cpu_virt_enabled(env))) {
 620         return -1;
 621     }
 622
 623     if (riscv_cpu_virt_enabled(env)) {
 624         if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
 625                                  0, mmu_idx, false, true)) {
 626             return -1;
 627         }
 628     }
 629
 630     return phys_addr & TARGET_PAGE_MASK;
 631 }
 632
 633 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
 634                                      vaddr addr, unsigned size,
 635                                      MMUAccessType access_type,
 636                                      int mmu_idx, MemTxAttrs attrs,
 637                                      MemTxResult response, uintptr_t retaddr)
 638 {
 639     RISCVCPU *cpu = RISCV_CPU(cs);
 640     CPURISCVState *env = &cpu->env;
 641
 642     if (access_type == MMU_DATA_STORE) {
 643         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 644     } else {
 645         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
 646     }
 647
 648     env->badaddr = addr;
 649     riscv_raise_exception(&cpu->env, cs->exception_index, retaddr);
 650 }
 651
 652 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
 653                                    MMUAccessType access_type, int mmu_idx,
 654                                    uintptr_t retaddr)
 655 {
 656     RISCVCPU *cpu = RISCV_CPU(cs);
 657     CPURISCVState *env = &cpu->env;
 658     switch (access_type) {
 659     case MMU_INST_FETCH:
 660         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
 661         break;
 662     case MMU_DATA_LOAD:
 663         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
 664         break;
 665     case MMU_DATA_STORE:
 666         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
 667         break;
 668     default:
 669         g_assert_not_reached();
 670     }
 671     env->badaddr = addr;
 672     riscv_raise_exception(env, cs->exception_index, retaddr);
 673 }
 674 #endif /* !CONFIG_USER_ONLY */
 675
 676 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
 677                         MMUAccessType access_type, int mmu_idx,
 678                         bool probe, uintptr_t retaddr)
 679 {
 680     RISCVCPU *cpu = RISCV_CPU(cs);
 681     CPURISCVState *env = &cpu->env;
 682 #ifndef CONFIG_USER_ONLY
 683     vaddr im_address;
 684     hwaddr pa = 0;
 685     int prot, prot2;
 686     bool pmp_violation = false;
 687     bool first_stage_error = true;
 688     bool two_stage_lookup = false;
 689     int ret = TRANSLATE_FAIL;
 690     int mode = mmu_idx;
 691     target_ulong tlb_size = 0;
 692
 693     env->guest_phys_fault_addr = 0;
 694
 695     qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
 696                   __func__, address, access_type, mmu_idx);
 697
 698     if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 699         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 700             mode = get_field(env->mstatus, MSTATUS_MPP);
 701         }
 702     }
 703
 704     if (riscv_has_ext(env, RVH) && env->priv == PRV_M &&
 705         access_type != MMU_INST_FETCH &&
 706         get_field(env->mstatus, MSTATUS_MPRV) &&
 707         get_field(env->mstatus, MSTATUS_MPV)) {
 708         two_stage_lookup = true;
 709     }
 710
 711     if (riscv_cpu_virt_enabled(env) ||
 712         ((riscv_cpu_two_stage_lookup(mmu_idx) || two_stage_lookup) &&
 713          access_type != MMU_INST_FETCH)) {
 714         /* Two stage lookup */
 715         ret = get_physical_address(env, &pa, &prot, address,
 716                                    &env->guest_phys_fault_addr, access_type,
 717                                    mmu_idx, true, true);
 718
 719         /*
 720          * A G-stage exception may be triggered during two state lookup.
 721          * And the env->guest_phys_fault_addr has already been set in
 722          * get_physical_address().
 723          */
 724         if (ret == TRANSLATE_G_STAGE_FAIL) {
 725             first_stage_error = false;
 726             access_type = MMU_DATA_LOAD;
 727         }
 728
 729         qemu_log_mask(CPU_LOG_MMU,
 730                       "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
 731                       TARGET_FMT_plx " prot %d\n",
 732                       __func__, address, ret, pa, prot);
 733
 734         if (ret == TRANSLATE_SUCCESS) {
 735             /* Second stage lookup */
 736             im_address = pa;
 737
 738             ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
 739                                        access_type, mmu_idx, false, true);
 740
 741             qemu_log_mask(CPU_LOG_MMU,
 742                     "%s 2nd-stage address=%" VADDR_PRIx " ret %d physical "
 743                     TARGET_FMT_plx " prot %d\n",
 744                     __func__, im_address, ret, pa, prot2);
 745
 746             prot &= prot2;
 747
 748             if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 749                 (ret == TRANSLATE_SUCCESS) &&
 750                 !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
 751                 ret = TRANSLATE_PMP_FAIL;
 752             }
 753
 754             if (ret != TRANSLATE_SUCCESS) {
 755                 /*
 756                  * Guest physical address translation failed, this is a HS
 757                  * level exception
 758                  */
 759                 first_stage_error = false;
 760                 env->guest_phys_fault_addr = (im_address |
 761                                               (address &
 762                                                (TARGET_PAGE_SIZE - 1))) >> 2;
 763             }
 764         }
 765     } else {
 766         /* Single stage lookup */
 767         ret = get_physical_address(env, &pa, &prot, address, NULL,
 768                                    access_type, mmu_idx, true, false);
 769
 770         qemu_log_mask(CPU_LOG_MMU,
 771                       "%s address=%" VADDR_PRIx " ret %d physical "
 772                       TARGET_FMT_plx " prot %d\n",
 773                       __func__, address, ret, pa, prot);
 774     }
 775
 776     if (riscv_feature(env, RISCV_FEATURE_PMP) &&
 777         (ret == TRANSLATE_SUCCESS) &&
 778         !pmp_hart_has_privs(env, pa, size, 1 << access_type, mode)) {
 779         ret = TRANSLATE_PMP_FAIL;
 780     }
 781     if (ret == TRANSLATE_PMP_FAIL) {
 782         pmp_violation = true;
 783     }
 784
 785     if (ret == TRANSLATE_SUCCESS) {
 786         if (pmp_is_range_in_tlb(env, pa & TARGET_PAGE_MASK, &tlb_size)) {
 787             tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
 788                          prot, mmu_idx, tlb_size);
 789         } else {
 790             tlb_set_page(cs, address & TARGET_PAGE_MASK, pa & TARGET_PAGE_MASK,
 791                          prot, mmu_idx, TARGET_PAGE_SIZE);
 792         }
 793         return true;
 794     } else if (probe) {
 795         return false;
 796     } else {
 797         raise_mmu_exception(env, address, access_type, pmp_violation,
 798                             first_stage_error,
 799                             riscv_cpu_virt_enabled(env) ||
 800                                 riscv_cpu_two_stage_lookup(mmu_idx));
 801         riscv_raise_exception(env, cs->exception_index, retaddr);
 802     }
 803
 804     return true;
 805
 806 #else
 807     switch (access_type) {
 808     case MMU_INST_FETCH:
 809         cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
 810         break;
 811     case MMU_DATA_LOAD:
 812         cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
 813         break;
 814     case MMU_DATA_STORE:
 815         cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
 816         break;
 817     default:
 818         g_assert_not_reached();
 819     }
 820     env->badaddr = address;
 821     cpu_loop_exit_restore(cs, retaddr);
 822 #endif
 823 }
 824
 825 /*
 826  * Handle Traps
 827  *
 828  * Adapted from Spike's processor_t::take_trap.
 829  *
 830  */
 831 void riscv_cpu_do_interrupt(CPUState *cs)
 832 {
 833 #if !defined(CONFIG_USER_ONLY)
 834
 835     RISCVCPU *cpu = RISCV_CPU(cs);
 836     CPURISCVState *env = &cpu->env;
 837     bool force_hs_execp = riscv_cpu_force_hs_excep_enabled(env);
 838     uint64_t s;
 839
 840     /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
 841      * so we mask off the MSB and separate into trap type and cause.
 842      */
 843     bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
 844     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
 845     target_ulong deleg = async ? env->mideleg : env->medeleg;
 846     bool write_tval = false;
 847     target_ulong tval = 0;
 848     target_ulong htval = 0;
 849     target_ulong mtval2 = 0;
 850
 851     if  (cause == RISCV_EXCP_SEMIHOST) {
 852         if (env->priv >= PRV_S) {
 853             env->gpr[xA0] = do_common_semihosting(cs);
 854             env->pc += 4;
 855             return;
 856         }
 857         cause = RISCV_EXCP_BREAKPOINT;
 858     }
 859
 860     if (!async) {
 861         /* set tval to badaddr for traps with address information */
 862         switch (cause) {
 863         case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
 864         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
 865         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
 866             force_hs_execp = true;
 867             /* fallthrough */
 868         case RISCV_EXCP_INST_ADDR_MIS:
 869         case RISCV_EXCP_INST_ACCESS_FAULT:
 870         case RISCV_EXCP_LOAD_ADDR_MIS:
 871         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
 872         case RISCV_EXCP_LOAD_ACCESS_FAULT:
 873         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
 874         case RISCV_EXCP_INST_PAGE_FAULT:
 875         case RISCV_EXCP_LOAD_PAGE_FAULT:
 876         case RISCV_EXCP_STORE_PAGE_FAULT:
 877             write_tval  = true;
 878             tval = env->badaddr;
 879             break;
 880         default:
 881             break;
 882         }
 883         /* ecall is dispatched as one cause so translate based on mode */
 884         if (cause == RISCV_EXCP_U_ECALL) {
 885             assert(env->priv <= 3);
 886
 887             if (env->priv == PRV_M) {
 888                 cause = RISCV_EXCP_M_ECALL;
 889             } else if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
 890                 cause = RISCV_EXCP_VS_ECALL;
 891             } else if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) {
 892                 cause = RISCV_EXCP_S_ECALL;
 893             } else if (env->priv == PRV_U) {
 894                 cause = RISCV_EXCP_U_ECALL;
 895             }
 896         }
 897     }
 898
 899     trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
 900                      riscv_cpu_get_trap_name(cause, async));
 901
 902     qemu_log_mask(CPU_LOG_INT,
 903                   "%s: hart:"TARGET_FMT_ld", async:%d, cause:"TARGET_FMT_lx", "
 904                   "epc:0x"TARGET_FMT_lx", tval:0x"TARGET_FMT_lx", desc=%s\n",
 905                   __func__, env->mhartid, async, cause, env->pc, tval,
 906                   riscv_cpu_get_trap_name(cause, async));
 907
 908     if (env->priv <= PRV_S &&
 909             cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
 910         /* handle the trap in S-mode */
 911         if (riscv_has_ext(env, RVH)) {
 912             target_ulong hdeleg = async ? env->hideleg : env->hedeleg;
 913             bool two_stage_lookup = false;
 914
 915             if (env->priv == PRV_M ||
 916                 (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) ||
 917                 (env->priv == PRV_U && !riscv_cpu_virt_enabled(env) &&
 918                     get_field(env->hstatus, HSTATUS_HU))) {
 919                     two_stage_lookup = true;
 920             }
 921
 922             if ((riscv_cpu_virt_enabled(env) || two_stage_lookup) && write_tval) {
 923                 /*
 924                  * If we are writing a guest virtual address to stval, set
 925                  * this to 1. If we are trapping to VS we will set this to 0
 926                  * later.
 927                  */
 928                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 1);
 929             } else {
 930                 /* For other HS-mode traps, we set this to 0. */
 931                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
 932             }
 933
 934             if (riscv_cpu_virt_enabled(env) && ((hdeleg >> cause) & 1) &&
 935                 !force_hs_execp) {
 936                 /* Trap to VS mode */
 937                 /*
 938                  * See if we need to adjust cause. Yes if its VS mode interrupt
 939                  * no if hypervisor has delegated one of hs mode's interrupt
 940                  */
 941                 if (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
 942                     cause == IRQ_VS_EXT) {
 943                     cause = cause - 1;
 944                 }
 945                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
 946             } else if (riscv_cpu_virt_enabled(env)) {
 947                 /* Trap into HS mode, from virt */
 948                 riscv_cpu_swap_hypervisor_regs(env);
 949                 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
 950                                          env->priv);
 951                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 952                                          riscv_cpu_virt_enabled(env));
 953
 954                 htval = env->guest_phys_fault_addr;
 955
 956                 riscv_cpu_set_virt_enabled(env, 0);
 957                 riscv_cpu_set_force_hs_excep(env, 0);
 958             } else {
 959                 /* Trap into HS mode */
 960                 if (!two_stage_lookup) {
 961                     env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
 962                                              riscv_cpu_virt_enabled(env));
 963                 }
 964                 htval = env->guest_phys_fault_addr;
 965             }
 966         }
 967
 968         s = env->mstatus;
 969         s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
 970         s = set_field(s, MSTATUS_SPP, env->priv);
 971         s = set_field(s, MSTATUS_SIE, 0);
 972         env->mstatus = s;
 973         env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
 974         env->sepc = env->pc;
 975         env->sbadaddr = tval;
 976         env->htval = htval;
 977         env->pc = (env->stvec >> 2 << 2) +
 978             ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
 979         riscv_cpu_set_mode(env, PRV_S);
 980     } else {
 981         /* handle the trap in M-mode */
 982         if (riscv_has_ext(env, RVH)) {
 983             if (riscv_cpu_virt_enabled(env)) {
 984                 riscv_cpu_swap_hypervisor_regs(env);
 985             }
 986             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
 987                                      riscv_cpu_virt_enabled(env));
 988             if (riscv_cpu_virt_enabled(env) && tval) {
 989                 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
 990             }
 991
 992             mtval2 = env->guest_phys_fault_addr;
 993
 994             /* Trapping to M mode, virt is disabled */
 995             riscv_cpu_set_virt_enabled(env, 0);
 996             riscv_cpu_set_force_hs_excep(env, 0);
 997         }
 998
 999         s = env->mstatus;
1000         s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
1001         s = set_field(s, MSTATUS_MPP, env->priv);
1002         s = set_field(s, MSTATUS_MIE, 0);
1003         env->mstatus = s;
1004         env->mcause = cause | ~(((target_ulong)-1) >> async);
1005         env->mepc = env->pc;
1006         env->mbadaddr = tval;
1007         env->mtval2 = mtval2;
1008         env->pc = (env->mtvec >> 2 << 2) +
1009             ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
1010         riscv_cpu_set_mode(env, PRV_M);
1011     }
1012
1013     /* NOTE: it is not necessary to yield load reservations here. It is only
1014      * necessary for an SC from "another hart" to cause a load reservation
1015      * to be yielded. Refer to the memory consistency model section of the
1016      * RISC-V ISA Specification.
1017      */
1018
1019 #endif
1020     cs->exception_index = EXCP_NONE; /* mark handled to qemu */
1021 }