target/riscv/cpu_helper.c

   1 /*
   2  * RISC-V CPU helpers for qemu.
   3  *
   4  * Copyright (c) 2016-2017 Sagar Karandikar, sagark@eecs.berkeley.edu
   5  * Copyright (c) 2017-2018 SiFive, Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2 or later, as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  *
  16  * You should have received a copy of the GNU General Public License along with
  17  * this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include "qemu/osdep.h"
  21 #include "qemu/log.h"
  22 #include "qemu/main-loop.h"
  23 #include "cpu.h"
  24 #include "exec/exec-all.h"
  25 #include "tcg/tcg-op.h"
  26 #include "trace.h"
  27 #include "semihosting/common-semi.h"
  28
  29 int riscv_cpu_mmu_index(CPURISCVState *env, bool ifetch)
  30 {
  31 #ifdef CONFIG_USER_ONLY
  32     return 0;
  33 #else
  34     return env->priv;
  35 #endif
  36 }
  37
  38 #ifndef CONFIG_USER_ONLY
  39 static int riscv_cpu_local_irq_pending(CPURISCVState *env)
  40 {
  41     target_ulong irqs;
  42
  43     target_ulong mstatus_mie = get_field(env->mstatus, MSTATUS_MIE);
  44     target_ulong mstatus_sie = get_field(env->mstatus, MSTATUS_SIE);
  45     target_ulong hs_mstatus_sie = get_field(env->mstatus_hs, MSTATUS_SIE);
  46
  47     target_ulong pending = env->mip & env->mie &
  48                                ~(MIP_VSSIP | MIP_VSTIP | MIP_VSEIP);
  49     target_ulong vspending = (env->mip & env->mie &
  50                               (MIP_VSSIP | MIP_VSTIP | MIP_VSEIP));
  51
  52     target_ulong mie    = env->priv < PRV_M ||
  53                           (env->priv == PRV_M && mstatus_mie);
  54     target_ulong sie    = env->priv < PRV_S ||
  55                           (env->priv == PRV_S && mstatus_sie);
  56     target_ulong hs_sie = env->priv < PRV_S ||
  57                           (env->priv == PRV_S && hs_mstatus_sie);
  58
  59     if (riscv_cpu_virt_enabled(env)) {
  60         target_ulong pending_hs_irq = pending & -hs_sie;
  61
  62         if (pending_hs_irq) {
  63             riscv_cpu_set_force_hs_excep(env, FORCE_HS_EXCEP);
  64             return ctz64(pending_hs_irq);
  65         }
  66
  67         pending = vspending;
  68     }
  69
  70     irqs = (pending & ~env->mideleg & -mie) | (pending &  env->mideleg & -sie);
  71
  72     if (irqs) {
  73         return ctz64(irqs); /* since non-zero */
  74     } else {
  75         return RISCV_EXCP_NONE; /* indicates no pending interrupt */
  76     }
  77 }
  78 #endif
  79
  80 bool riscv_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
  81 {
  82 #if !defined(CONFIG_USER_ONLY)
  83     if (interrupt_request & CPU_INTERRUPT_HARD) {
  84         RISCVCPU *cpu = RISCV_CPU(cs);
  85         CPURISCVState *env = &cpu->env;
  86         int interruptno = riscv_cpu_local_irq_pending(env);
  87         if (interruptno >= 0) {
  88             cs->exception_index = RISCV_EXCP_INT_FLAG | interruptno;
  89             riscv_cpu_do_interrupt(cs);
  90             return true;
  91         }
  92     }
  93 #endif
  94     return false;
  95 }
  96
  97 #if !defined(CONFIG_USER_ONLY)
  98
  99 /* Return true is floating point support is currently enabled */
 100 bool riscv_cpu_fp_enabled(CPURISCVState *env)
 101 {
 102     if (env->mstatus & MSTATUS_FS) {
 103         if (riscv_cpu_virt_enabled(env) && !(env->mstatus_hs & MSTATUS_FS)) {
 104             return false;
 105         }
 106         return true;
 107     }
 108
 109     return false;
 110 }
 111
 112 void riscv_cpu_swap_hypervisor_regs(CPURISCVState *env)
 113 {
 114     uint64_t mstatus_mask = MSTATUS_MXR | MSTATUS_SUM | MSTATUS_FS |
 115                             MSTATUS_SPP | MSTATUS_SPIE | MSTATUS_SIE |
 116                             MSTATUS64_UXL;
 117     bool current_virt = riscv_cpu_virt_enabled(env);
 118
 119     g_assert(riscv_has_ext(env, RVH));
 120
 121     if (current_virt) {
 122         /* Current V=1 and we are about to change to V=0 */
 123         env->vsstatus = env->mstatus & mstatus_mask;
 124         env->mstatus &= ~mstatus_mask;
 125         env->mstatus |= env->mstatus_hs;
 126
 127         env->vstvec = env->stvec;
 128         env->stvec = env->stvec_hs;
 129
 130         env->vsscratch = env->sscratch;
 131         env->sscratch = env->sscratch_hs;
 132
 133         env->vsepc = env->sepc;
 134         env->sepc = env->sepc_hs;
 135
 136         env->vscause = env->scause;
 137         env->scause = env->scause_hs;
 138
 139         env->vstval = env->stval;
 140         env->stval = env->stval_hs;
 141
 142         env->vsatp = env->satp;
 143         env->satp = env->satp_hs;
 144     } else {
 145         /* Current V=0 and we are about to change to V=1 */
 146         env->mstatus_hs = env->mstatus & mstatus_mask;
 147         env->mstatus &= ~mstatus_mask;
 148         env->mstatus |= env->vsstatus;
 149
 150         env->stvec_hs = env->stvec;
 151         env->stvec = env->vstvec;
 152
 153         env->sscratch_hs = env->sscratch;
 154         env->sscratch = env->vsscratch;
 155
 156         env->sepc_hs = env->sepc;
 157         env->sepc = env->vsepc;
 158
 159         env->scause_hs = env->scause;
 160         env->scause = env->vscause;
 161
 162         env->stval_hs = env->stval;
 163         env->stval = env->vstval;
 164
 165         env->satp_hs = env->satp;
 166         env->satp = env->vsatp;
 167     }
 168 }
 169
 170 bool riscv_cpu_virt_enabled(CPURISCVState *env)
 171 {
 172     if (!riscv_has_ext(env, RVH)) {
 173         return false;
 174     }
 175
 176     return get_field(env->virt, VIRT_ONOFF);
 177 }
 178
 179 void riscv_cpu_set_virt_enabled(CPURISCVState *env, bool enable)
 180 {
 181     if (!riscv_has_ext(env, RVH)) {
 182         return;
 183     }
 184
 185     /* Flush the TLB on all virt mode changes. */
 186     if (get_field(env->virt, VIRT_ONOFF) != enable) {
 187         tlb_flush(env_cpu(env));
 188     }
 189
 190     env->virt = set_field(env->virt, VIRT_ONOFF, enable);
 191 }
 192
 193 bool riscv_cpu_force_hs_excep_enabled(CPURISCVState *env)
 194 {
 195     if (!riscv_has_ext(env, RVH)) {
 196         return false;
 197     }
 198
 199     return get_field(env->virt, FORCE_HS_EXCEP);
 200 }
 201
 202 void riscv_cpu_set_force_hs_excep(CPURISCVState *env, bool enable)
 203 {
 204     if (!riscv_has_ext(env, RVH)) {
 205         return;
 206     }
 207
 208     env->virt = set_field(env->virt, FORCE_HS_EXCEP, enable);
 209 }
 210
 211 bool riscv_cpu_two_stage_lookup(int mmu_idx)
 212 {
 213     return mmu_idx & TB_FLAGS_PRIV_HYP_ACCESS_MASK;
 214 }
 215
 216 int riscv_cpu_claim_interrupts(RISCVCPU *cpu, uint32_t interrupts)
 217 {
 218     CPURISCVState *env = &cpu->env;
 219     if (env->miclaim & interrupts) {
 220         return -1;
 221     } else {
 222         env->miclaim |= interrupts;
 223         return 0;
 224     }
 225 }
 226
 227 uint32_t riscv_cpu_update_mip(RISCVCPU *cpu, uint32_t mask, uint32_t value)
 228 {
 229     CPURISCVState *env = &cpu->env;
 230     CPUState *cs = CPU(cpu);
 231     uint32_t old = env->mip;
 232     bool locked = false;
 233
 234     if (!qemu_mutex_iothread_locked()) {
 235         locked = true;
 236         qemu_mutex_lock_iothread();
 237     }
 238
 239     env->mip = (env->mip & ~mask) | (value & mask);
 240
 241     if (env->mip) {
 242         cpu_interrupt(cs, CPU_INTERRUPT_HARD);
 243     } else {
 244         cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD);
 245     }
 246
 247     if (locked) {
 248         qemu_mutex_unlock_iothread();
 249     }
 250
 251     return old;
 252 }
 253
 254 void riscv_cpu_set_rdtime_fn(CPURISCVState *env, uint64_t (*fn)(uint32_t),
 255                              uint32_t arg)
 256 {
 257     env->rdtime_fn = fn;
 258     env->rdtime_fn_arg = arg;
 259 }
 260
 261 void riscv_cpu_set_mode(CPURISCVState *env, target_ulong newpriv)
 262 {
 263     if (newpriv > PRV_M) {
 264         g_assert_not_reached();
 265     }
 266     if (newpriv == PRV_H) {
 267         newpriv = PRV_U;
 268     }
 269     /* tlb_flush is unnecessary as mode is contained in mmu_idx */
 270     env->priv = newpriv;
 271
 272     /*
 273      * Clear the load reservation - otherwise a reservation placed in one
 274      * context/process can be used by another, resulting in an SC succeeding
 275      * incorrectly. Version 2.2 of the ISA specification explicitly requires
 276      * this behaviour, while later revisions say that the kernel "should" use
 277      * an SC instruction to force the yielding of a load reservation on a
 278      * preemptive context switch. As a result, do both.
 279      */
 280     env->load_res = -1;
 281 }
 282
 283 /*
 284  * get_physical_address_pmp - check PMP permission for this physical address
 285  *
 286  * Match the PMP region and check permission for this physical address and it's
 287  * TLB page. Returns 0 if the permission checking was successful
 288  *
 289  * @env: CPURISCVState
 290  * @prot: The returned protection attributes
 291  * @tlb_size: TLB page size containing addr. It could be modified after PMP
 292  *            permission checking. NULL if not set TLB page for addr.
 293  * @addr: The physical address to be checked permission
 294  * @access_type: The type of MMU access
 295  * @mode: Indicates current privilege level.
 296  */
 297 static int get_physical_address_pmp(CPURISCVState *env, int *prot,
 298                                     target_ulong *tlb_size, hwaddr addr,
 299                                     int size, MMUAccessType access_type,
 300                                     int mode)
 301 {
 302     pmp_priv_t pmp_priv;
 303     target_ulong tlb_size_pmp = 0;
 304
 305     if (!riscv_feature(env, RISCV_FEATURE_PMP)) {
 306         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 307         return TRANSLATE_SUCCESS;
 308     }
 309
 310     if (!pmp_hart_has_privs(env, addr, size, 1 << access_type, &pmp_priv,
 311                             mode)) {
 312         *prot = 0;
 313         return TRANSLATE_PMP_FAIL;
 314     }
 315
 316     *prot = pmp_priv_to_page_prot(pmp_priv);
 317     if (tlb_size != NULL) {
 318         if (pmp_is_range_in_tlb(env, addr & ~(*tlb_size - 1), &tlb_size_pmp)) {
 319             *tlb_size = tlb_size_pmp;
 320         }
 321     }
 322
 323     return TRANSLATE_SUCCESS;
 324 }
 325
 326 /* get_physical_address - get the physical address for this virtual address
 327  *
 328  * Do a page table walk to obtain the physical address corresponding to a
 329  * virtual address. Returns 0 if the translation was successful
 330  *
 331  * Adapted from Spike's mmu_t::translate and mmu_t::walk
 332  *
 333  * @env: CPURISCVState
 334  * @physical: This will be set to the calculated physical address
 335  * @prot: The returned protection attributes
 336  * @addr: The virtual address to be translated
 337  * @fault_pte_addr: If not NULL, this will be set to fault pte address
 338  *                  when a error occurs on pte address translation.
 339  *                  This will already be shifted to match htval.
 340  * @access_type: The type of MMU access
 341  * @mmu_idx: Indicates current privilege level
 342  * @first_stage: Are we in first stage translation?
 343  *               Second stage is used for hypervisor guest translation
 344  * @two_stage: Are we going to perform two stage translation
 345  * @is_debug: Is this access from a debugger or the monitor?
 346  */
 347 static int get_physical_address(CPURISCVState *env, hwaddr *physical,
 348                                 int *prot, target_ulong addr,
 349                                 target_ulong *fault_pte_addr,
 350                                 int access_type, int mmu_idx,
 351                                 bool first_stage, bool two_stage,
 352                                 bool is_debug)
 353 {
 354     /* NOTE: the env->pc value visible here will not be
 355      * correct, but the value visible to the exception handler
 356      * (riscv_cpu_do_interrupt) is correct */
 357     MemTxResult res;
 358     MemTxAttrs attrs = MEMTXATTRS_UNSPECIFIED;
 359     int mode = mmu_idx & TB_FLAGS_PRIV_MMU_MASK;
 360     bool use_background = false;
 361
 362     /*
 363      * Check if we should use the background registers for the two
 364      * stage translation. We don't need to check if we actually need
 365      * two stage translation as that happened before this function
 366      * was called. Background registers will be used if the guest has
 367      * forced a two stage translation to be on (in HS or M mode).
 368      */
 369     if (!riscv_cpu_virt_enabled(env) && two_stage) {
 370         use_background = true;
 371     }
 372
 373     /* MPRV does not affect the virtual-machine load/store
 374        instructions, HLV, HLVX, and HSV. */
 375     if (riscv_cpu_two_stage_lookup(mmu_idx)) {
 376         mode = get_field(env->hstatus, HSTATUS_SPVP);
 377     } else if (mode == PRV_M && access_type != MMU_INST_FETCH) {
 378         if (get_field(env->mstatus, MSTATUS_MPRV)) {
 379             mode = get_field(env->mstatus, MSTATUS_MPP);
 380         }
 381     }
 382
 383     if (first_stage == false) {
 384         /* We are in stage 2 translation, this is similar to stage 1. */
 385         /* Stage 2 is always taken as U-mode */
 386         mode = PRV_U;
 387     }
 388
 389     if (mode == PRV_M || !riscv_feature(env, RISCV_FEATURE_MMU)) {
 390         *physical = addr;
 391         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 392         return TRANSLATE_SUCCESS;
 393     }
 394
 395     *prot = 0;
 396
 397     hwaddr base;
 398     int levels, ptidxbits, ptesize, vm, sum, mxr, widened;
 399
 400     if (first_stage == true) {
 401         mxr = get_field(env->mstatus, MSTATUS_MXR);
 402     } else {
 403         mxr = get_field(env->vsstatus, MSTATUS_MXR);
 404     }
 405
 406     if (first_stage == true) {
 407         if (use_background) {
 408             if (riscv_cpu_is_32bit(env)) {
 409                 base = (hwaddr)get_field(env->vsatp, SATP32_PPN) << PGSHIFT;
 410                 vm = get_field(env->vsatp, SATP32_MODE);
 411             } else {
 412                 base = (hwaddr)get_field(env->vsatp, SATP64_PPN) << PGSHIFT;
 413                 vm = get_field(env->vsatp, SATP64_MODE);
 414             }
 415         } else {
 416             if (riscv_cpu_is_32bit(env)) {
 417                 base = (hwaddr)get_field(env->satp, SATP32_PPN) << PGSHIFT;
 418                 vm = get_field(env->satp, SATP32_MODE);
 419             } else {
 420                 base = (hwaddr)get_field(env->satp, SATP64_PPN) << PGSHIFT;
 421                 vm = get_field(env->satp, SATP64_MODE);
 422             }
 423         }
 424         widened = 0;
 425     } else {
 426         if (riscv_cpu_is_32bit(env)) {
 427             base = (hwaddr)get_field(env->hgatp, SATP32_PPN) << PGSHIFT;
 428             vm = get_field(env->hgatp, SATP32_MODE);
 429         } else {
 430             base = (hwaddr)get_field(env->hgatp, SATP64_PPN) << PGSHIFT;
 431             vm = get_field(env->hgatp, SATP64_MODE);
 432         }
 433         widened = 2;
 434     }
 435     /* status.SUM will be ignored if execute on background */
 436     sum = get_field(env->mstatus, MSTATUS_SUM) || use_background || is_debug;
 437     switch (vm) {
 438     case VM_1_10_SV32:
 439       levels = 2; ptidxbits = 10; ptesize = 4; break;
 440     case VM_1_10_SV39:
 441       levels = 3; ptidxbits = 9; ptesize = 8; break;
 442     case VM_1_10_SV48:
 443       levels = 4; ptidxbits = 9; ptesize = 8; break;
 444     case VM_1_10_SV57:
 445       levels = 5; ptidxbits = 9; ptesize = 8; break;
 446     case VM_1_10_MBARE:
 447         *physical = addr;
 448         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 449         return TRANSLATE_SUCCESS;
 450     default:
 451       g_assert_not_reached();
 452     }
 453
 454     CPUState *cs = env_cpu(env);
 455     int va_bits = PGSHIFT + levels * ptidxbits + widened;
 456     target_ulong mask, masked_msbs;
 457
 458     if (TARGET_LONG_BITS > (va_bits - 1)) {
 459         mask = (1L << (TARGET_LONG_BITS - (va_bits - 1))) - 1;
 460     } else {
 461         mask = 0;
 462     }
 463     masked_msbs = (addr >> (va_bits - 1)) & mask;
 464
 465     if (masked_msbs != 0 && masked_msbs != mask) {
 466         return TRANSLATE_FAIL;
 467     }
 468
 469     int ptshift = (levels - 1) * ptidxbits;
 470     int i;
 471
 472 #if !TCG_OVERSIZED_GUEST
 473 restart:
 474 #endif
 475     for (i = 0; i < levels; i++, ptshift -= ptidxbits) {
 476         target_ulong idx;
 477         if (i == 0) {
 478             idx = (addr >> (PGSHIFT + ptshift)) &
 479                            ((1 << (ptidxbits + widened)) - 1);
 480         } else {
 481             idx = (addr >> (PGSHIFT + ptshift)) &
 482                            ((1 << ptidxbits) - 1);
 483         }
 484
 485         /* check that physical address of PTE is legal */
 486         hwaddr pte_addr;
 487
 488         if (two_stage && first_stage) {
 489             int vbase_prot;
 490             hwaddr vbase;
 491
 492             /* Do the second stage translation on the base PTE address. */
 493             int vbase_ret = get_physical_address(env, &vbase, &vbase_prot,
 494                                                  base, NULL, MMU_DATA_LOAD,
 495                                                  mmu_idx, false, true,
 496                                                  is_debug);
 497
 498             if (vbase_ret != TRANSLATE_SUCCESS) {
 499                 if (fault_pte_addr) {
 500                     *fault_pte_addr = (base + idx * ptesize) >> 2;
 501                 }
 502                 return TRANSLATE_G_STAGE_FAIL;
 503             }
 504
 505             pte_addr = vbase + idx * ptesize;
 506         } else {
 507             pte_addr = base + idx * ptesize;
 508         }
 509
 510         int pmp_prot;
 511         int pmp_ret = get_physical_address_pmp(env, &pmp_prot, NULL, pte_addr,
 512                                                sizeof(target_ulong),
 513                                                MMU_DATA_LOAD, PRV_S);
 514         if (pmp_ret != TRANSLATE_SUCCESS) {
 515             return TRANSLATE_PMP_FAIL;
 516         }
 517
 518         target_ulong pte;
 519         if (riscv_cpu_is_32bit(env)) {
 520             pte = address_space_ldl(cs->as, pte_addr, attrs, &res);
 521         } else {
 522             pte = address_space_ldq(cs->as, pte_addr, attrs, &res);
 523         }
 524
 525         if (res != MEMTX_OK) {
 526             return TRANSLATE_FAIL;
 527         }
 528
 529         hwaddr ppn = pte >> PTE_PPN_SHIFT;
 530
 531         if (!(pte & PTE_V)) {
 532             /* Invalid PTE */
 533             return TRANSLATE_FAIL;
 534         } else if (!(pte & (PTE_R | PTE_W | PTE_X))) {
 535             /* Inner PTE, continue walking */
 536             base = ppn << PGSHIFT;
 537         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == PTE_W) {
 538             /* Reserved leaf PTE flags: PTE_W */
 539             return TRANSLATE_FAIL;
 540         } else if ((pte & (PTE_R | PTE_W | PTE_X)) == (PTE_W | PTE_X)) {
 541             /* Reserved leaf PTE flags: PTE_W + PTE_X */
 542             return TRANSLATE_FAIL;
 543         } else if ((pte & PTE_U) && ((mode != PRV_U) &&
 544                    (!sum || access_type == MMU_INST_FETCH))) {
 545             /* User PTE flags when not U mode and mstatus.SUM is not set,
 546                or the access type is an instruction fetch */
 547             return TRANSLATE_FAIL;
 548         } else if (!(pte & PTE_U) && (mode != PRV_S)) {
 549             /* Supervisor PTE flags when not S mode */
 550             return TRANSLATE_FAIL;
 551         } else if (ppn & ((1ULL << ptshift) - 1)) {
 552             /* Misaligned PPN */
 553             return TRANSLATE_FAIL;
 554         } else if (access_type == MMU_DATA_LOAD && !((pte & PTE_R) ||
 555                    ((pte & PTE_X) && mxr))) {
 556             /* Read access check failed */
 557             return TRANSLATE_FAIL;
 558         } else if (access_type == MMU_DATA_STORE && !(pte & PTE_W)) {
 559             /* Write access check failed */
 560             return TRANSLATE_FAIL;
 561         } else if (access_type == MMU_INST_FETCH && !(pte & PTE_X)) {
 562             /* Fetch access check failed */
 563             return TRANSLATE_FAIL;
 564         } else {
 565             /* if necessary, set accessed and dirty bits. */
 566             target_ulong updated_pte = pte | PTE_A |
 567                 (access_type == MMU_DATA_STORE ? PTE_D : 0);
 568
 569             /* Page table updates need to be atomic with MTTCG enabled */
 570             if (updated_pte != pte) {
 571                 /*
 572                  * - if accessed or dirty bits need updating, and the PTE is
 573                  *   in RAM, then we do so atomically with a compare and swap.
 574                  * - if the PTE is in IO space or ROM, then it can't be updated
 575                  *   and we return TRANSLATE_FAIL.
 576                  * - if the PTE changed by the time we went to update it, then
 577                  *   it is no longer valid and we must re-walk the page table.
 578                  */
 579                 MemoryRegion *mr;
 580                 hwaddr l = sizeof(target_ulong), addr1;
 581                 mr = address_space_translate(cs->as, pte_addr,
 582                     &addr1, &l, false, MEMTXATTRS_UNSPECIFIED);
 583                 if (memory_region_is_ram(mr)) {
 584                     target_ulong *pte_pa =
 585                         qemu_map_ram_ptr(mr->ram_block, addr1);
 586 #if TCG_OVERSIZED_GUEST
 587                     /* MTTCG is not enabled on oversized TCG guests so
 588                      * page table updates do not need to be atomic */
 589                     *pte_pa = pte = updated_pte;
 590 #else
 591                     target_ulong old_pte =
 592                         qatomic_cmpxchg(pte_pa, pte, updated_pte);
 593                     if (old_pte != pte) {
 594                         goto restart;
 595                     } else {
 596                         pte = updated_pte;
 597                     }
 598 #endif
 599                 } else {
 600                     /* misconfigured PTE in ROM (AD bits are not preset) or
 601                      * PTE is in IO space and can't be updated atomically */
 602                     return TRANSLATE_FAIL;
 603                 }
 604             }
 605
 606             /* for superpage mappings, make a fake leaf PTE for the TLB's
 607                benefit. */
 608             target_ulong vpn = addr >> PGSHIFT;
 609             *physical = ((ppn | (vpn & ((1L << ptshift) - 1))) << PGSHIFT) |
 610                         (addr & ~TARGET_PAGE_MASK);
 611
 612             /* set permissions on the TLB entry */
 613             if ((pte & PTE_R) || ((pte & PTE_X) && mxr)) {
 614                 *prot |= PAGE_READ;
 615             }
 616             if ((pte & PTE_X)) {
 617                 *prot |= PAGE_EXEC;
 618             }
 619             /* add write permission on stores or if the page is already dirty,
 620                so that we TLB miss on later writes to update the dirty bit */
 621             if ((pte & PTE_W) &&
 622                     (access_type == MMU_DATA_STORE || (pte & PTE_D))) {
 623                 *prot |= PAGE_WRITE;
 624             }
 625             return TRANSLATE_SUCCESS;
 626         }
 627     }
 628     return TRANSLATE_FAIL;
 629 }
 630
 631 static void raise_mmu_exception(CPURISCVState *env, target_ulong address,
 632                                 MMUAccessType access_type, bool pmp_violation,
 633                                 bool first_stage, bool two_stage)
 634 {
 635     CPUState *cs = env_cpu(env);
 636     int page_fault_exceptions, vm;
 637     uint64_t stap_mode;
 638
 639     if (riscv_cpu_is_32bit(env)) {
 640         stap_mode = SATP32_MODE;
 641     } else {
 642         stap_mode = SATP64_MODE;
 643     }
 644
 645     if (first_stage) {
 646         vm = get_field(env->satp, stap_mode);
 647     } else {
 648         vm = get_field(env->hgatp, stap_mode);
 649     }
 650
 651     page_fault_exceptions = vm != VM_1_10_MBARE && !pmp_violation;
 652
 653     switch (access_type) {
 654     case MMU_INST_FETCH:
 655         if (riscv_cpu_virt_enabled(env) && !first_stage) {
 656             cs->exception_index = RISCV_EXCP_INST_GUEST_PAGE_FAULT;
 657         } else {
 658             cs->exception_index = page_fault_exceptions ?
 659                 RISCV_EXCP_INST_PAGE_FAULT : RISCV_EXCP_INST_ACCESS_FAULT;
 660         }
 661         break;
 662     case MMU_DATA_LOAD:
 663         if (two_stage && !first_stage) {
 664             cs->exception_index = RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT;
 665         } else {
 666             cs->exception_index = page_fault_exceptions ?
 667                 RISCV_EXCP_LOAD_PAGE_FAULT : RISCV_EXCP_LOAD_ACCESS_FAULT;
 668         }
 669         break;
 670     case MMU_DATA_STORE:
 671         if (two_stage && !first_stage) {
 672             cs->exception_index = RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT;
 673         } else {
 674             cs->exception_index = page_fault_exceptions ?
 675                 RISCV_EXCP_STORE_PAGE_FAULT : RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 676         }
 677         break;
 678     default:
 679         g_assert_not_reached();
 680     }
 681     env->badaddr = address;
 682     env->two_stage_lookup = two_stage;
 683 }
 684
 685 hwaddr riscv_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
 686 {
 687     RISCVCPU *cpu = RISCV_CPU(cs);
 688     CPURISCVState *env = &cpu->env;
 689     hwaddr phys_addr;
 690     int prot;
 691     int mmu_idx = cpu_mmu_index(&cpu->env, false);
 692
 693     if (get_physical_address(env, &phys_addr, &prot, addr, NULL, 0, mmu_idx,
 694                              true, riscv_cpu_virt_enabled(env), true)) {
 695         return -1;
 696     }
 697
 698     if (riscv_cpu_virt_enabled(env)) {
 699         if (get_physical_address(env, &phys_addr, &prot, phys_addr, NULL,
 700                                  0, mmu_idx, false, true, true)) {
 701             return -1;
 702         }
 703     }
 704
 705     return phys_addr & TARGET_PAGE_MASK;
 706 }
 707
 708 void riscv_cpu_do_transaction_failed(CPUState *cs, hwaddr physaddr,
 709                                      vaddr addr, unsigned size,
 710                                      MMUAccessType access_type,
 711                                      int mmu_idx, MemTxAttrs attrs,
 712                                      MemTxResult response, uintptr_t retaddr)
 713 {
 714     RISCVCPU *cpu = RISCV_CPU(cs);
 715     CPURISCVState *env = &cpu->env;
 716
 717     if (access_type == MMU_DATA_STORE) {
 718         cs->exception_index = RISCV_EXCP_STORE_AMO_ACCESS_FAULT;
 719     } else if (access_type == MMU_DATA_LOAD) {
 720         cs->exception_index = RISCV_EXCP_LOAD_ACCESS_FAULT;
 721     } else {
 722         cs->exception_index = RISCV_EXCP_INST_ACCESS_FAULT;
 723     }
 724
 725     env->badaddr = addr;
 726     env->two_stage_lookup = riscv_cpu_virt_enabled(env) ||
 727                             riscv_cpu_two_stage_lookup(mmu_idx);
 728     riscv_raise_exception(&cpu->env, cs->exception_index, retaddr);
 729 }
 730
 731 void riscv_cpu_do_unaligned_access(CPUState *cs, vaddr addr,
 732                                    MMUAccessType access_type, int mmu_idx,
 733                                    uintptr_t retaddr)
 734 {
 735     RISCVCPU *cpu = RISCV_CPU(cs);
 736     CPURISCVState *env = &cpu->env;
 737     switch (access_type) {
 738     case MMU_INST_FETCH:
 739         cs->exception_index = RISCV_EXCP_INST_ADDR_MIS;
 740         break;
 741     case MMU_DATA_LOAD:
 742         cs->exception_index = RISCV_EXCP_LOAD_ADDR_MIS;
 743         break;
 744     case MMU_DATA_STORE:
 745         cs->exception_index = RISCV_EXCP_STORE_AMO_ADDR_MIS;
 746         break;
 747     default:
 748         g_assert_not_reached();
 749     }
 750     env->badaddr = addr;
 751     env->two_stage_lookup = riscv_cpu_virt_enabled(env) ||
 752                             riscv_cpu_two_stage_lookup(mmu_idx);
 753     riscv_raise_exception(env, cs->exception_index, retaddr);
 754 }
 755 #endif /* !CONFIG_USER_ONLY */
 756
 757 bool riscv_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
 758                         MMUAccessType access_type, int mmu_idx,
 759                         bool probe, uintptr_t retaddr)
 760 {
 761     RISCVCPU *cpu = RISCV_CPU(cs);
 762     CPURISCVState *env = &cpu->env;
 763 #ifndef CONFIG_USER_ONLY
 764     vaddr im_address;
 765     hwaddr pa = 0;
 766     int prot, prot2, prot_pmp;
 767     bool pmp_violation = false;
 768     bool first_stage_error = true;
 769     bool two_stage_lookup = false;
 770     int ret = TRANSLATE_FAIL;
 771     int mode = mmu_idx;
 772     /* default TLB page size */
 773     target_ulong tlb_size = TARGET_PAGE_SIZE;
 774
 775     env->guest_phys_fault_addr = 0;
 776
 777     qemu_log_mask(CPU_LOG_MMU, "%s ad %" VADDR_PRIx " rw %d mmu_idx %d\n",
 778                   __func__, address, access_type, mmu_idx);
 779
 780     /* MPRV does not affect the virtual-machine load/store
 781        instructions, HLV, HLVX, and HSV. */
 782     if (riscv_cpu_two_stage_lookup(mmu_idx)) {
 783         mode = get_field(env->hstatus, HSTATUS_SPVP);
 784     } else if (mode == PRV_M && access_type != MMU_INST_FETCH &&
 785                get_field(env->mstatus, MSTATUS_MPRV)) {
 786         mode = get_field(env->mstatus, MSTATUS_MPP);
 787         if (riscv_has_ext(env, RVH) && get_field(env->mstatus, MSTATUS_MPV)) {
 788             two_stage_lookup = true;
 789         }
 790     }
 791
 792     if (riscv_cpu_virt_enabled(env) ||
 793         ((riscv_cpu_two_stage_lookup(mmu_idx) || two_stage_lookup) &&
 794          access_type != MMU_INST_FETCH)) {
 795         /* Two stage lookup */
 796         ret = get_physical_address(env, &pa, &prot, address,
 797                                    &env->guest_phys_fault_addr, access_type,
 798                                    mmu_idx, true, true, false);
 799
 800         /*
 801          * A G-stage exception may be triggered during two state lookup.
 802          * And the env->guest_phys_fault_addr has already been set in
 803          * get_physical_address().
 804          */
 805         if (ret == TRANSLATE_G_STAGE_FAIL) {
 806             first_stage_error = false;
 807             access_type = MMU_DATA_LOAD;
 808         }
 809
 810         qemu_log_mask(CPU_LOG_MMU,
 811                       "%s 1st-stage address=%" VADDR_PRIx " ret %d physical "
 812                       TARGET_FMT_plx " prot %d\n",
 813                       __func__, address, ret, pa, prot);
 814
 815         if (ret == TRANSLATE_SUCCESS) {
 816             /* Second stage lookup */
 817             im_address = pa;
 818
 819             ret = get_physical_address(env, &pa, &prot2, im_address, NULL,
 820                                        access_type, mmu_idx, false, true,
 821                                        false);
 822
 823             qemu_log_mask(CPU_LOG_MMU,
 824                     "%s 2nd-stage address=%" VADDR_PRIx " ret %d physical "
 825                     TARGET_FMT_plx " prot %d\n",
 826                     __func__, im_address, ret, pa, prot2);
 827
 828             prot &= prot2;
 829
 830             if (ret == TRANSLATE_SUCCESS) {
 831                 ret = get_physical_address_pmp(env, &prot_pmp, &tlb_size, pa,
 832                                                size, access_type, mode);
 833
 834                 qemu_log_mask(CPU_LOG_MMU,
 835                               "%s PMP address=" TARGET_FMT_plx " ret %d prot"
 836                               " %d tlb_size " TARGET_FMT_lu "\n",
 837                               __func__, pa, ret, prot_pmp, tlb_size);
 838
 839                 prot &= prot_pmp;
 840             }
 841
 842             if (ret != TRANSLATE_SUCCESS) {
 843                 /*
 844                  * Guest physical address translation failed, this is a HS
 845                  * level exception
 846                  */
 847                 first_stage_error = false;
 848                 env->guest_phys_fault_addr = (im_address |
 849                                               (address &
 850                                                (TARGET_PAGE_SIZE - 1))) >> 2;
 851             }
 852         }
 853     } else {
 854         /* Single stage lookup */
 855         ret = get_physical_address(env, &pa, &prot, address, NULL,
 856                                    access_type, mmu_idx, true, false, false);
 857
 858         qemu_log_mask(CPU_LOG_MMU,
 859                       "%s address=%" VADDR_PRIx " ret %d physical "
 860                       TARGET_FMT_plx " prot %d\n",
 861                       __func__, address, ret, pa, prot);
 862
 863         if (ret == TRANSLATE_SUCCESS) {
 864             ret = get_physical_address_pmp(env, &prot_pmp, &tlb_size, pa,
 865                                            size, access_type, mode);
 866
 867             qemu_log_mask(CPU_LOG_MMU,
 868                           "%s PMP address=" TARGET_FMT_plx " ret %d prot"
 869                           " %d tlb_size " TARGET_FMT_lu "\n",
 870                           __func__, pa, ret, prot_pmp, tlb_size);
 871
 872             prot &= prot_pmp;
 873         }
 874     }
 875
 876     if (ret == TRANSLATE_PMP_FAIL) {
 877         pmp_violation = true;
 878     }
 879
 880     if (ret == TRANSLATE_SUCCESS) {
 881         tlb_set_page(cs, address & ~(tlb_size - 1), pa & ~(tlb_size - 1),
 882                      prot, mmu_idx, tlb_size);
 883         return true;
 884     } else if (probe) {
 885         return false;
 886     } else {
 887         raise_mmu_exception(env, address, access_type, pmp_violation,
 888                             first_stage_error,
 889                             riscv_cpu_virt_enabled(env) ||
 890                                 riscv_cpu_two_stage_lookup(mmu_idx));
 891         riscv_raise_exception(env, cs->exception_index, retaddr);
 892     }
 893
 894     return true;
 895
 896 #else
 897     switch (access_type) {
 898     case MMU_INST_FETCH:
 899         cs->exception_index = RISCV_EXCP_INST_PAGE_FAULT;
 900         break;
 901     case MMU_DATA_LOAD:
 902         cs->exception_index = RISCV_EXCP_LOAD_PAGE_FAULT;
 903         break;
 904     case MMU_DATA_STORE:
 905         cs->exception_index = RISCV_EXCP_STORE_PAGE_FAULT;
 906         break;
 907     default:
 908         g_assert_not_reached();
 909     }
 910     env->badaddr = address;
 911     cpu_loop_exit_restore(cs, retaddr);
 912 #endif
 913 }
 914
 915 /*
 916  * Handle Traps
 917  *
 918  * Adapted from Spike's processor_t::take_trap.
 919  *
 920  */
 921 void riscv_cpu_do_interrupt(CPUState *cs)
 922 {
 923 #if !defined(CONFIG_USER_ONLY)
 924
 925     RISCVCPU *cpu = RISCV_CPU(cs);
 926     CPURISCVState *env = &cpu->env;
 927     bool force_hs_execp = riscv_cpu_force_hs_excep_enabled(env);
 928     uint64_t s;
 929
 930     /* cs->exception is 32-bits wide unlike mcause which is XLEN-bits wide
 931      * so we mask off the MSB and separate into trap type and cause.
 932      */
 933     bool async = !!(cs->exception_index & RISCV_EXCP_INT_FLAG);
 934     target_ulong cause = cs->exception_index & RISCV_EXCP_INT_MASK;
 935     target_ulong deleg = async ? env->mideleg : env->medeleg;
 936     bool write_tval = false;
 937     target_ulong tval = 0;
 938     target_ulong htval = 0;
 939     target_ulong mtval2 = 0;
 940
 941     if  (cause == RISCV_EXCP_SEMIHOST) {
 942         if (env->priv >= PRV_S) {
 943             env->gpr[xA0] = do_common_semihosting(cs);
 944             env->pc += 4;
 945             return;
 946         }
 947         cause = RISCV_EXCP_BREAKPOINT;
 948     }
 949
 950     if (!async) {
 951         /* set tval to badaddr for traps with address information */
 952         switch (cause) {
 953         case RISCV_EXCP_INST_GUEST_PAGE_FAULT:
 954         case RISCV_EXCP_LOAD_GUEST_ACCESS_FAULT:
 955         case RISCV_EXCP_STORE_GUEST_AMO_ACCESS_FAULT:
 956             force_hs_execp = true;
 957             /* fallthrough */
 958         case RISCV_EXCP_INST_ADDR_MIS:
 959         case RISCV_EXCP_INST_ACCESS_FAULT:
 960         case RISCV_EXCP_LOAD_ADDR_MIS:
 961         case RISCV_EXCP_STORE_AMO_ADDR_MIS:
 962         case RISCV_EXCP_LOAD_ACCESS_FAULT:
 963         case RISCV_EXCP_STORE_AMO_ACCESS_FAULT:
 964         case RISCV_EXCP_INST_PAGE_FAULT:
 965         case RISCV_EXCP_LOAD_PAGE_FAULT:
 966         case RISCV_EXCP_STORE_PAGE_FAULT:
 967             write_tval  = true;
 968             tval = env->badaddr;
 969             break;
 970         default:
 971             break;
 972         }
 973         /* ecall is dispatched as one cause so translate based on mode */
 974         if (cause == RISCV_EXCP_U_ECALL) {
 975             assert(env->priv <= 3);
 976
 977             if (env->priv == PRV_M) {
 978                 cause = RISCV_EXCP_M_ECALL;
 979             } else if (env->priv == PRV_S && riscv_cpu_virt_enabled(env)) {
 980                 cause = RISCV_EXCP_VS_ECALL;
 981             } else if (env->priv == PRV_S && !riscv_cpu_virt_enabled(env)) {
 982                 cause = RISCV_EXCP_S_ECALL;
 983             } else if (env->priv == PRV_U) {
 984                 cause = RISCV_EXCP_U_ECALL;
 985             }
 986         }
 987     }
 988
 989     trace_riscv_trap(env->mhartid, async, cause, env->pc, tval,
 990                      riscv_cpu_get_trap_name(cause, async));
 991
 992     qemu_log_mask(CPU_LOG_INT,
 993                   "%s: hart:"TARGET_FMT_ld", async:%d, cause:"TARGET_FMT_lx", "
 994                   "epc:0x"TARGET_FMT_lx", tval:0x"TARGET_FMT_lx", desc=%s\n",
 995                   __func__, env->mhartid, async, cause, env->pc, tval,
 996                   riscv_cpu_get_trap_name(cause, async));
 997
 998     if (env->priv <= PRV_S &&
 999             cause < TARGET_LONG_BITS && ((deleg >> cause) & 1)) {
1000         /* handle the trap in S-mode */
1001         if (riscv_has_ext(env, RVH)) {
1002             target_ulong hdeleg = async ? env->hideleg : env->hedeleg;
1003
1004             if (env->two_stage_lookup && write_tval) {
1005                 /*
1006                  * If we are writing a guest virtual address to stval, set
1007                  * this to 1. If we are trapping to VS we will set this to 0
1008                  * later.
1009                  */
1010                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 1);
1011             } else {
1012                 /* For other HS-mode traps, we set this to 0. */
1013                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
1014             }
1015
1016             if (riscv_cpu_virt_enabled(env) && ((hdeleg >> cause) & 1) &&
1017                 !force_hs_execp) {
1018                 /* Trap to VS mode */
1019                 /*
1020                  * See if we need to adjust cause. Yes if its VS mode interrupt
1021                  * no if hypervisor has delegated one of hs mode's interrupt
1022                  */
1023                 if (cause == IRQ_VS_TIMER || cause == IRQ_VS_SOFT ||
1024                     cause == IRQ_VS_EXT) {
1025                     cause = cause - 1;
1026                 }
1027                 env->hstatus = set_field(env->hstatus, HSTATUS_GVA, 0);
1028             } else if (riscv_cpu_virt_enabled(env)) {
1029                 /* Trap into HS mode, from virt */
1030                 riscv_cpu_swap_hypervisor_regs(env);
1031                 env->hstatus = set_field(env->hstatus, HSTATUS_SPVP,
1032                                          env->priv);
1033                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV,
1034                                          riscv_cpu_virt_enabled(env));
1035
1036                 htval = env->guest_phys_fault_addr;
1037
1038                 riscv_cpu_set_virt_enabled(env, 0);
1039                 riscv_cpu_set_force_hs_excep(env, 0);
1040             } else {
1041                 /* Trap into HS mode */
1042                 env->hstatus = set_field(env->hstatus, HSTATUS_SPV, false);
1043                 htval = env->guest_phys_fault_addr;
1044             }
1045         }
1046
1047         s = env->mstatus;
1048         s = set_field(s, MSTATUS_SPIE, get_field(s, MSTATUS_SIE));
1049         s = set_field(s, MSTATUS_SPP, env->priv);
1050         s = set_field(s, MSTATUS_SIE, 0);
1051         env->mstatus = s;
1052         env->scause = cause | ((target_ulong)async << (TARGET_LONG_BITS - 1));
1053         env->sepc = env->pc;
1054         env->stval = tval;
1055         env->htval = htval;
1056         env->pc = (env->stvec >> 2 << 2) +
1057             ((async && (env->stvec & 3) == 1) ? cause * 4 : 0);
1058         riscv_cpu_set_mode(env, PRV_S);
1059     } else {
1060         /* handle the trap in M-mode */
1061         if (riscv_has_ext(env, RVH)) {
1062             if (riscv_cpu_virt_enabled(env)) {
1063                 riscv_cpu_swap_hypervisor_regs(env);
1064             }
1065             env->mstatus = set_field(env->mstatus, MSTATUS_MPV,
1066                                      riscv_cpu_virt_enabled(env));
1067             if (riscv_cpu_virt_enabled(env) && tval) {
1068                 env->mstatus = set_field(env->mstatus, MSTATUS_GVA, 1);
1069             }
1070
1071             mtval2 = env->guest_phys_fault_addr;
1072
1073             /* Trapping to M mode, virt is disabled */
1074             riscv_cpu_set_virt_enabled(env, 0);
1075             riscv_cpu_set_force_hs_excep(env, 0);
1076         }
1077
1078         s = env->mstatus;
1079         s = set_field(s, MSTATUS_MPIE, get_field(s, MSTATUS_MIE));
1080         s = set_field(s, MSTATUS_MPP, env->priv);
1081         s = set_field(s, MSTATUS_MIE, 0);
1082         env->mstatus = s;
1083         env->mcause = cause | ~(((target_ulong)-1) >> async);
1084         env->mepc = env->pc;
1085         env->mtval = tval;
1086         env->mtval2 = mtval2;
1087         env->pc = (env->mtvec >> 2 << 2) +
1088             ((async && (env->mtvec & 3) == 1) ? cause * 4 : 0);
1089         riscv_cpu_set_mode(env, PRV_M);
1090     }
1091
1092     /* NOTE: it is not necessary to yield load reservations here. It is only
1093      * necessary for an SC from "another hart" to cause a load reservation
1094      * to be yielded. Refer to the memory consistency model section of the
1095      * RISC-V ISA Specification.
1096      */
1097
1098     env->two_stage_lookup = false;
1099 #endif
1100     cs->exception_index = RISCV_EXCP_NONE; /* mark handled to qemu */
1101 }