target/mips/tcg/sysemu/tlb_helper.c

   1 /*
   2  * MIPS TLB (Translation lookaside buffer) helpers.
   3  *
   4  *  Copyright (c) 2004-2005 Jocelyn Mayer
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "qemu/osdep.h"
  20 #include "qemu/bitops.h"
  21
  22 #include "cpu.h"
  23 #include "internal.h"
  24 #include "exec/exec-all.h"
  25 #include "exec/page-protection.h"
  26 #include "exec/cpu_ldst.h"
  27 #include "exec/log.h"
  28 #include "exec/helper-proto.h"
  29
  30 /* TLB management */
  31 static void r4k_mips_tlb_flush_extra(CPUMIPSState *env, int first)
  32 {
  33     /* Discard entries from env->tlb[first] onwards.  */
  34     while (env->tlb->tlb_in_use > first) {
  35         r4k_invalidate_tlb(env, --env->tlb->tlb_in_use, 0);
  36     }
  37 }
  38
  39 static inline uint64_t get_tlb_pfn_from_entrylo(uint64_t entrylo)
  40 {
  41 #if defined(TARGET_MIPS64)
  42     return extract64(entrylo, 6, 54);
  43 #else
  44     return extract64(entrylo, 6, 24) | /* PFN */
  45            (extract64(entrylo, 32, 32) << 24); /* PFNX */
  46 #endif
  47 }
  48
  49 static void r4k_fill_tlb(CPUMIPSState *env, int idx)
  50 {
  51     r4k_tlb_t *tlb;
  52     uint64_t mask = env->CP0_PageMask >> (TARGET_PAGE_BITS + 1);
  53
  54     /* XXX: detect conflicting TLBs and raise a MCHECK exception when needed */
  55     tlb = &env->tlb->mmu.r4k.tlb[idx];
  56     if (env->CP0_EntryHi & (1 << CP0EnHi_EHINV)) {
  57         tlb->EHINV = 1;
  58         return;
  59     }
  60     tlb->EHINV = 0;
  61     tlb->VPN = env->CP0_EntryHi & (TARGET_PAGE_MASK << 1);
  62 #if defined(TARGET_MIPS64)
  63     tlb->VPN &= env->SEGMask;
  64 #endif
  65     tlb->ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
  66     tlb->MMID = env->CP0_MemoryMapID;
  67     tlb->PageMask = env->CP0_PageMask;
  68     tlb->G = env->CP0_EntryLo0 & env->CP0_EntryLo1 & 1;
  69     tlb->V0 = (env->CP0_EntryLo0 & 2) != 0;
  70     tlb->D0 = (env->CP0_EntryLo0 & 4) != 0;
  71     tlb->C0 = (env->CP0_EntryLo0 >> 3) & 0x7;
  72     tlb->XI0 = (env->CP0_EntryLo0 >> CP0EnLo_XI) & 1;
  73     tlb->RI0 = (env->CP0_EntryLo0 >> CP0EnLo_RI) & 1;
  74     tlb->PFN[0] = (get_tlb_pfn_from_entrylo(env->CP0_EntryLo0) & ~mask) << 12;
  75     tlb->V1 = (env->CP0_EntryLo1 & 2) != 0;
  76     tlb->D1 = (env->CP0_EntryLo1 & 4) != 0;
  77     tlb->C1 = (env->CP0_EntryLo1 >> 3) & 0x7;
  78     tlb->XI1 = (env->CP0_EntryLo1 >> CP0EnLo_XI) & 1;
  79     tlb->RI1 = (env->CP0_EntryLo1 >> CP0EnLo_RI) & 1;
  80     tlb->PFN[1] = (get_tlb_pfn_from_entrylo(env->CP0_EntryLo1) & ~mask) << 12;
  81 }
  82
  83 static void r4k_helper_tlbinv(CPUMIPSState *env)
  84 {
  85     bool mi = !!((env->CP0_Config5 >> CP0C5_MI) & 1);
  86     uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
  87     uint32_t MMID = env->CP0_MemoryMapID;
  88     uint32_t tlb_mmid;
  89     r4k_tlb_t *tlb;
  90     int idx;
  91
  92     MMID = mi ? MMID : (uint32_t) ASID;
  93     for (idx = 0; idx < env->tlb->nb_tlb; idx++) {
  94         tlb = &env->tlb->mmu.r4k.tlb[idx];
  95         tlb_mmid = mi ? tlb->MMID : (uint32_t) tlb->ASID;
  96         if (!tlb->G && tlb_mmid == MMID) {
  97             tlb->EHINV = 1;
  98         }
  99     }
 100     cpu_mips_tlb_flush(env);
 101 }
 102
 103 static void r4k_helper_tlbinvf(CPUMIPSState *env)
 104 {
 105     int idx;
 106
 107     for (idx = 0; idx < env->tlb->nb_tlb; idx++) {
 108         env->tlb->mmu.r4k.tlb[idx].EHINV = 1;
 109     }
 110     cpu_mips_tlb_flush(env);
 111 }
 112
 113 static void r4k_helper_tlbwi(CPUMIPSState *env)
 114 {
 115     bool mi = !!((env->CP0_Config5 >> CP0C5_MI) & 1);
 116     target_ulong VPN;
 117     uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
 118     uint32_t MMID = env->CP0_MemoryMapID;
 119     uint32_t tlb_mmid;
 120     bool EHINV, G, V0, D0, V1, D1, XI0, XI1, RI0, RI1;
 121     r4k_tlb_t *tlb;
 122     int idx;
 123
 124     MMID = mi ? MMID : (uint32_t) ASID;
 125
 126     idx = (env->CP0_Index & ~0x80000000) % env->tlb->nb_tlb;
 127     tlb = &env->tlb->mmu.r4k.tlb[idx];
 128     VPN = env->CP0_EntryHi & (TARGET_PAGE_MASK << 1);
 129 #if defined(TARGET_MIPS64)
 130     VPN &= env->SEGMask;
 131 #endif
 132     EHINV = (env->CP0_EntryHi & (1 << CP0EnHi_EHINV)) != 0;
 133     G = env->CP0_EntryLo0 & env->CP0_EntryLo1 & 1;
 134     V0 = (env->CP0_EntryLo0 & 2) != 0;
 135     D0 = (env->CP0_EntryLo0 & 4) != 0;
 136     XI0 = (env->CP0_EntryLo0 >> CP0EnLo_XI) &1;
 137     RI0 = (env->CP0_EntryLo0 >> CP0EnLo_RI) &1;
 138     V1 = (env->CP0_EntryLo1 & 2) != 0;
 139     D1 = (env->CP0_EntryLo1 & 4) != 0;
 140     XI1 = (env->CP0_EntryLo1 >> CP0EnLo_XI) &1;
 141     RI1 = (env->CP0_EntryLo1 >> CP0EnLo_RI) &1;
 142
 143     tlb_mmid = mi ? tlb->MMID : (uint32_t) tlb->ASID;
 144     /*
 145      * Discard cached TLB entries, unless tlbwi is just upgrading access
 146      * permissions on the current entry.
 147      */
 148     if (tlb->VPN != VPN || tlb_mmid != MMID || tlb->G != G ||
 149         (!tlb->EHINV && EHINV) ||
 150         (tlb->V0 && !V0) || (tlb->D0 && !D0) ||
 151         (!tlb->XI0 && XI0) || (!tlb->RI0 && RI0) ||
 152         (tlb->V1 && !V1) || (tlb->D1 && !D1) ||
 153         (!tlb->XI1 && XI1) || (!tlb->RI1 && RI1)) {
 154         r4k_mips_tlb_flush_extra(env, env->tlb->nb_tlb);
 155     }
 156
 157     r4k_invalidate_tlb(env, idx, 0);
 158     r4k_fill_tlb(env, idx);
 159 }
 160
 161 static void r4k_helper_tlbwr(CPUMIPSState *env)
 162 {
 163     int r = cpu_mips_get_random(env);
 164
 165     r4k_invalidate_tlb(env, r, 1);
 166     r4k_fill_tlb(env, r);
 167 }
 168
 169 static void r4k_helper_tlbp(CPUMIPSState *env)
 170 {
 171     bool mi = !!((env->CP0_Config5 >> CP0C5_MI) & 1);
 172     r4k_tlb_t *tlb;
 173     target_ulong mask;
 174     target_ulong tag;
 175     target_ulong VPN;
 176     uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
 177     uint32_t MMID = env->CP0_MemoryMapID;
 178     uint32_t tlb_mmid;
 179     int i;
 180
 181     MMID = mi ? MMID : (uint32_t) ASID;
 182     for (i = 0; i < env->tlb->nb_tlb; i++) {
 183         tlb = &env->tlb->mmu.r4k.tlb[i];
 184         /* 1k pages are not supported. */
 185         mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
 186         tag = env->CP0_EntryHi & ~mask;
 187         VPN = tlb->VPN & ~mask;
 188 #if defined(TARGET_MIPS64)
 189         tag &= env->SEGMask;
 190 #endif
 191         tlb_mmid = mi ? tlb->MMID : (uint32_t) tlb->ASID;
 192         /* Check ASID/MMID, virtual page number & size */
 193         if ((tlb->G == 1 || tlb_mmid == MMID) && VPN == tag && !tlb->EHINV) {
 194             /* TLB match */
 195             env->CP0_Index = i;
 196             break;
 197         }
 198     }
 199     if (i == env->tlb->nb_tlb) {
 200         /* No match.  Discard any shadow entries, if any of them match.  */
 201         for (i = env->tlb->nb_tlb; i < env->tlb->tlb_in_use; i++) {
 202             tlb = &env->tlb->mmu.r4k.tlb[i];
 203             /* 1k pages are not supported. */
 204             mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
 205             tag = env->CP0_EntryHi & ~mask;
 206             VPN = tlb->VPN & ~mask;
 207 #if defined(TARGET_MIPS64)
 208             tag &= env->SEGMask;
 209 #endif
 210             tlb_mmid = mi ? tlb->MMID : (uint32_t) tlb->ASID;
 211             /* Check ASID/MMID, virtual page number & size */
 212             if ((tlb->G == 1 || tlb_mmid == MMID) && VPN == tag) {
 213                 r4k_mips_tlb_flush_extra(env, i);
 214                 break;
 215             }
 216         }
 217
 218         env->CP0_Index |= 0x80000000;
 219     }
 220 }
 221
 222 static inline uint64_t get_entrylo_pfn_from_tlb(uint64_t tlb_pfn)
 223 {
 224 #if defined(TARGET_MIPS64)
 225     return tlb_pfn << 6;
 226 #else
 227     return (extract64(tlb_pfn, 0, 24) << 6) | /* PFN */
 228            (extract64(tlb_pfn, 24, 32) << 32); /* PFNX */
 229 #endif
 230 }
 231
 232 static void r4k_helper_tlbr(CPUMIPSState *env)
 233 {
 234     bool mi = !!((env->CP0_Config5 >> CP0C5_MI) & 1);
 235     uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
 236     uint32_t MMID = env->CP0_MemoryMapID;
 237     uint32_t tlb_mmid;
 238     r4k_tlb_t *tlb;
 239     int idx;
 240
 241     MMID = mi ? MMID : (uint32_t) ASID;
 242     idx = (env->CP0_Index & ~0x80000000) % env->tlb->nb_tlb;
 243     tlb = &env->tlb->mmu.r4k.tlb[idx];
 244
 245     tlb_mmid = mi ? tlb->MMID : (uint32_t) tlb->ASID;
 246     /* If this will change the current ASID/MMID, flush qemu's TLB.  */
 247     if (MMID != tlb_mmid) {
 248         cpu_mips_tlb_flush(env);
 249     }
 250
 251     r4k_mips_tlb_flush_extra(env, env->tlb->nb_tlb);
 252
 253     if (tlb->EHINV) {
 254         env->CP0_EntryHi = 1 << CP0EnHi_EHINV;
 255         env->CP0_PageMask = 0;
 256         env->CP0_EntryLo0 = 0;
 257         env->CP0_EntryLo1 = 0;
 258     } else {
 259         env->CP0_EntryHi = mi ? tlb->VPN : tlb->VPN | tlb->ASID;
 260         env->CP0_MemoryMapID = tlb->MMID;
 261         env->CP0_PageMask = tlb->PageMask;
 262         env->CP0_EntryLo0 = tlb->G | (tlb->V0 << 1) | (tlb->D0 << 2) |
 263                         ((uint64_t)tlb->RI0 << CP0EnLo_RI) |
 264                         ((uint64_t)tlb->XI0 << CP0EnLo_XI) | (tlb->C0 << 3) |
 265                         get_entrylo_pfn_from_tlb(tlb->PFN[0] >> 12);
 266         env->CP0_EntryLo1 = tlb->G | (tlb->V1 << 1) | (tlb->D1 << 2) |
 267                         ((uint64_t)tlb->RI1 << CP0EnLo_RI) |
 268                         ((uint64_t)tlb->XI1 << CP0EnLo_XI) | (tlb->C1 << 3) |
 269                         get_entrylo_pfn_from_tlb(tlb->PFN[1] >> 12);
 270     }
 271 }
 272
 273 void helper_tlbwi(CPUMIPSState *env)
 274 {
 275     env->tlb->helper_tlbwi(env);
 276 }
 277
 278 void helper_tlbwr(CPUMIPSState *env)
 279 {
 280     env->tlb->helper_tlbwr(env);
 281 }
 282
 283 void helper_tlbp(CPUMIPSState *env)
 284 {
 285     env->tlb->helper_tlbp(env);
 286 }
 287
 288 void helper_tlbr(CPUMIPSState *env)
 289 {
 290     env->tlb->helper_tlbr(env);
 291 }
 292
 293 void helper_tlbinv(CPUMIPSState *env)
 294 {
 295     env->tlb->helper_tlbinv(env);
 296 }
 297
 298 void helper_tlbinvf(CPUMIPSState *env)
 299 {
 300     env->tlb->helper_tlbinvf(env);
 301 }
 302
 303 static void global_invalidate_tlb(CPUMIPSState *env,
 304                            uint32_t invMsgVPN2,
 305                            uint8_t invMsgR,
 306                            uint32_t invMsgMMid,
 307                            bool invAll,
 308                            bool invVAMMid,
 309                            bool invMMid,
 310                            bool invVA)
 311 {
 312
 313     int idx;
 314     r4k_tlb_t *tlb;
 315     bool VAMatch;
 316     bool MMidMatch;
 317
 318     for (idx = 0; idx < env->tlb->nb_tlb; idx++) {
 319         tlb = &env->tlb->mmu.r4k.tlb[idx];
 320         VAMatch =
 321             (((tlb->VPN & ~tlb->PageMask) == (invMsgVPN2 & ~tlb->PageMask))
 322 #ifdef TARGET_MIPS64
 323             &&
 324             (extract64(env->CP0_EntryHi, 62, 2) == invMsgR)
 325 #endif
 326             );
 327         MMidMatch = tlb->MMID == invMsgMMid;
 328         if ((invAll && (idx > env->CP0_Wired)) ||
 329             (VAMatch && invVAMMid && (tlb->G || MMidMatch)) ||
 330             (VAMatch && invVA) ||
 331             (MMidMatch && !(tlb->G) && invMMid)) {
 332             tlb->EHINV = 1;
 333         }
 334     }
 335     cpu_mips_tlb_flush(env);
 336 }
 337
 338 void helper_ginvt(CPUMIPSState *env, target_ulong arg, uint32_t type)
 339 {
 340     bool invAll = type == 0;
 341     bool invVA = type == 1;
 342     bool invMMid = type == 2;
 343     bool invVAMMid = type == 3;
 344     uint32_t invMsgVPN2 = arg & (TARGET_PAGE_MASK << 1);
 345     uint8_t invMsgR = 0;
 346     uint32_t invMsgMMid = env->CP0_MemoryMapID;
 347     CPUState *other_cs = first_cpu;
 348
 349 #ifdef TARGET_MIPS64
 350     invMsgR = extract64(arg, 62, 2);
 351 #endif
 352
 353     CPU_FOREACH(other_cs) {
 354         MIPSCPU *other_cpu = MIPS_CPU(other_cs);
 355         global_invalidate_tlb(&other_cpu->env, invMsgVPN2, invMsgR, invMsgMMid,
 356                               invAll, invVAMMid, invMMid, invVA);
 357     }
 358 }
 359
 360 /* no MMU emulation */
 361 static int no_mmu_map_address(CPUMIPSState *env, hwaddr *physical, int *prot,
 362                               target_ulong address, MMUAccessType access_type)
 363 {
 364     *physical = address;
 365     *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 366     return TLBRET_MATCH;
 367 }
 368
 369 /* fixed mapping MMU emulation */
 370 static int fixed_mmu_map_address(CPUMIPSState *env, hwaddr *physical,
 371                                  int *prot, target_ulong address,
 372                                  MMUAccessType access_type)
 373 {
 374     if (address <= (int32_t)0x7FFFFFFFUL) {
 375         if (!(env->CP0_Status & (1 << CP0St_ERL))) {
 376             *physical = address + 0x40000000UL;
 377         } else {
 378             *physical = address;
 379         }
 380     } else if (address <= (int32_t)0xBFFFFFFFUL) {
 381         *physical = address & 0x1FFFFFFF;
 382     } else {
 383         *physical = address;
 384     }
 385
 386     *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
 387     return TLBRET_MATCH;
 388 }
 389
 390 /* MIPS32/MIPS64 R4000-style MMU emulation */
 391 static int r4k_map_address(CPUMIPSState *env, hwaddr *physical, int *prot,
 392                            target_ulong address, MMUAccessType access_type)
 393 {
 394     uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
 395     uint32_t MMID = env->CP0_MemoryMapID;
 396     bool mi = !!((env->CP0_Config5 >> CP0C5_MI) & 1);
 397     uint32_t tlb_mmid;
 398     int i;
 399
 400     MMID = mi ? MMID : (uint32_t) ASID;
 401
 402     for (i = 0; i < env->tlb->tlb_in_use; i++) {
 403         r4k_tlb_t *tlb = &env->tlb->mmu.r4k.tlb[i];
 404         /* 1k pages are not supported. */
 405         target_ulong mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
 406         target_ulong tag = address & ~mask;
 407         target_ulong VPN = tlb->VPN & ~mask;
 408 #if defined(TARGET_MIPS64)
 409         tag &= env->SEGMask;
 410 #endif
 411
 412         /* Check ASID/MMID, virtual page number & size */
 413         tlb_mmid = mi ? tlb->MMID : (uint32_t) tlb->ASID;
 414         if ((tlb->G == 1 || tlb_mmid == MMID) && VPN == tag && !tlb->EHINV) {
 415             /* TLB match */
 416             int n = !!(address & mask & ~(mask >> 1));
 417             /* Check access rights */
 418             if (!(n ? tlb->V1 : tlb->V0)) {
 419                 return TLBRET_INVALID;
 420             }
 421             if (access_type == MMU_INST_FETCH && (n ? tlb->XI1 : tlb->XI0)) {
 422                 return TLBRET_XI;
 423             }
 424             if (access_type == MMU_DATA_LOAD && (n ? tlb->RI1 : tlb->RI0)) {
 425                 return TLBRET_RI;
 426             }
 427             if (access_type != MMU_DATA_STORE || (n ? tlb->D1 : tlb->D0)) {
 428                 *physical = tlb->PFN[n] | (address & (mask >> 1));
 429                 *prot = PAGE_READ;
 430                 if (n ? tlb->D1 : tlb->D0) {
 431                     *prot |= PAGE_WRITE;
 432                 }
 433                 if (!(n ? tlb->XI1 : tlb->XI0)) {
 434                     *prot |= PAGE_EXEC;
 435                 }
 436                 return TLBRET_MATCH;
 437             }
 438             return TLBRET_DIRTY;
 439         }
 440     }
 441     return TLBRET_NOMATCH;
 442 }
 443
 444 static void no_mmu_init(CPUMIPSState *env, const mips_def_t *def)
 445 {
 446     env->tlb->nb_tlb = 1;
 447     env->tlb->map_address = &no_mmu_map_address;
 448 }
 449
 450 static void fixed_mmu_init(CPUMIPSState *env, const mips_def_t *def)
 451 {
 452     env->tlb->nb_tlb = 1;
 453     env->tlb->map_address = &fixed_mmu_map_address;
 454 }
 455
 456 static void r4k_mmu_init(CPUMIPSState *env, const mips_def_t *def)
 457 {
 458     env->tlb->nb_tlb = 1 + ((def->CP0_Config1 >> CP0C1_MMU) & 63);
 459     env->tlb->map_address = &r4k_map_address;
 460     env->tlb->helper_tlbwi = r4k_helper_tlbwi;
 461     env->tlb->helper_tlbwr = r4k_helper_tlbwr;
 462     env->tlb->helper_tlbp = r4k_helper_tlbp;
 463     env->tlb->helper_tlbr = r4k_helper_tlbr;
 464     env->tlb->helper_tlbinv = r4k_helper_tlbinv;
 465     env->tlb->helper_tlbinvf = r4k_helper_tlbinvf;
 466 }
 467
 468 void mmu_init(CPUMIPSState *env, const mips_def_t *def)
 469 {
 470     env->tlb = g_malloc0(sizeof(CPUMIPSTLBContext));
 471
 472     switch (def->mmu_type) {
 473     case MMU_TYPE_NONE:
 474         no_mmu_init(env, def);
 475         break;
 476     case MMU_TYPE_R4000:
 477         r4k_mmu_init(env, def);
 478         break;
 479     case MMU_TYPE_FMT:
 480         fixed_mmu_init(env, def);
 481         break;
 482     case MMU_TYPE_R3000:
 483     case MMU_TYPE_R6000:
 484     case MMU_TYPE_R8000:
 485     default:
 486         cpu_abort(env_cpu(env), "MMU type not supported\n");
 487     }
 488 }
 489
 490 void cpu_mips_tlb_flush(CPUMIPSState *env)
 491 {
 492     /* Flush qemu's TLB and discard all shadowed entries.  */
 493     tlb_flush(env_cpu(env));
 494     env->tlb->tlb_in_use = env->tlb->nb_tlb;
 495 }
 496
 497 static void raise_mmu_exception(CPUMIPSState *env, target_ulong address,
 498                                 MMUAccessType access_type, int tlb_error)
 499 {
 500     CPUState *cs = env_cpu(env);
 501     int exception = 0, error_code = 0;
 502
 503     if (access_type == MMU_INST_FETCH) {
 504         error_code |= EXCP_INST_NOTAVAIL;
 505     }
 506
 507     switch (tlb_error) {
 508     default:
 509     case TLBRET_BADADDR:
 510         /* Reference to kernel address from user mode or supervisor mode */
 511         /* Reference to supervisor address from user mode */
 512         if (access_type == MMU_DATA_STORE) {
 513             exception = EXCP_AdES;
 514         } else {
 515             exception = EXCP_AdEL;
 516         }
 517         break;
 518     case TLBRET_NOMATCH:
 519         /* No TLB match for a mapped address */
 520         if (access_type == MMU_DATA_STORE) {
 521             exception = EXCP_TLBS;
 522         } else {
 523             exception = EXCP_TLBL;
 524         }
 525         error_code |= EXCP_TLB_NOMATCH;
 526         break;
 527     case TLBRET_INVALID:
 528         /* TLB match with no valid bit */
 529         if (access_type == MMU_DATA_STORE) {
 530             exception = EXCP_TLBS;
 531         } else {
 532             exception = EXCP_TLBL;
 533         }
 534         break;
 535     case TLBRET_DIRTY:
 536         /* TLB match but 'D' bit is cleared */
 537         exception = EXCP_LTLBL;
 538         break;
 539     case TLBRET_XI:
 540         /* Execute-Inhibit Exception */
 541         if (env->CP0_PageGrain & (1 << CP0PG_IEC)) {
 542             exception = EXCP_TLBXI;
 543         } else {
 544             exception = EXCP_TLBL;
 545         }
 546         break;
 547     case TLBRET_RI:
 548         /* Read-Inhibit Exception */
 549         if (env->CP0_PageGrain & (1 << CP0PG_IEC)) {
 550             exception = EXCP_TLBRI;
 551         } else {
 552             exception = EXCP_TLBL;
 553         }
 554         break;
 555     }
 556     /* Raise exception */
 557     if (!(env->hflags & MIPS_HFLAG_DM)) {
 558         env->CP0_BadVAddr = address;
 559     }
 560     env->CP0_Context = (env->CP0_Context & ~0x007fffff) |
 561                        ((address >> 9) & 0x007ffff0);
 562     env->CP0_EntryHi = (env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask) |
 563                        (env->CP0_EntryHi & (1 << CP0EnHi_EHINV)) |
 564                        (address & (TARGET_PAGE_MASK << 1));
 565 #if defined(TARGET_MIPS64)
 566     env->CP0_EntryHi &= env->SEGMask;
 567     env->CP0_XContext =
 568         (env->CP0_XContext & ((~0ULL) << (env->SEGBITS - 7))) | /* PTEBase */
 569         (extract64(address, 62, 2) << (env->SEGBITS - 9)) |     /* R       */
 570         (extract64(address, 13, env->SEGBITS - 13) << 4);       /* BadVPN2 */
 571 #endif
 572     cs->exception_index = exception;
 573     env->error_code = error_code;
 574 }
 575
 576 #if !defined(TARGET_MIPS64)
 577
 578 /*
 579  * Perform hardware page table walk
 580  *
 581  * Memory accesses are performed using the KERNEL privilege level.
 582  * Synchronous exceptions detected on memory accesses cause a silent exit
 583  * from page table walking, resulting in a TLB or XTLB Refill exception.
 584  *
 585  * Implementations are not required to support page table walk memory
 586  * accesses from mapped memory regions. When an unsupported access is
 587  * attempted, a silent exit is taken, resulting in a TLB or XTLB Refill
 588  * exception.
 589  *
 590  * Note that if an exception is caused by AddressTranslation or LoadMemory
 591  * functions, the exception is not taken, a silent exit is taken,
 592  * resulting in a TLB or XTLB Refill exception.
 593  */
 594
 595 static bool get_pte(CPUMIPSState *env, uint64_t vaddr, int entry_size,
 596         uint64_t *pte)
 597 {
 598     if ((vaddr & ((entry_size >> 3) - 1)) != 0) {
 599         return false;
 600     }
 601     if (entry_size == 64) {
 602         *pte = cpu_ldq_code(env, vaddr);
 603     } else {
 604         *pte = cpu_ldl_code(env, vaddr);
 605     }
 606     return true;
 607 }
 608
 609 static uint64_t get_tlb_entry_layout(CPUMIPSState *env, uint64_t entry,
 610         int entry_size, int ptei)
 611 {
 612     uint64_t result = entry;
 613     uint64_t rixi;
 614     if (ptei > entry_size) {
 615         ptei -= 32;
 616     }
 617     result >>= (ptei - 2);
 618     rixi = result & 3;
 619     result >>= 2;
 620     result |= rixi << CP0EnLo_XI;
 621     return result;
 622 }
 623
 624 static int walk_directory(CPUMIPSState *env, uint64_t *vaddr,
 625         int directory_index, bool *huge_page, bool *hgpg_directory_hit,
 626         uint64_t *pw_entrylo0, uint64_t *pw_entrylo1,
 627         unsigned directory_shift, unsigned leaf_shift, int ptw_mmu_idx)
 628 {
 629     int dph = (env->CP0_PWCtl >> CP0PC_DPH) & 0x1;
 630     int psn = (env->CP0_PWCtl >> CP0PC_PSN) & 0x3F;
 631     int hugepg = (env->CP0_PWCtl >> CP0PC_HUGEPG) & 0x1;
 632     int pf_ptew = (env->CP0_PWField >> CP0PF_PTEW) & 0x3F;
 633     uint32_t direntry_size = 1 << (directory_shift + 3);
 634     uint32_t leafentry_size = 1 << (leaf_shift + 3);
 635     uint64_t entry;
 636     uint64_t paddr;
 637     int prot;
 638     uint64_t lsb = 0;
 639     uint64_t w = 0;
 640
 641     if (get_physical_address(env, &paddr, &prot, *vaddr, MMU_DATA_LOAD,
 642                              ptw_mmu_idx) != TLBRET_MATCH) {
 643         /* wrong base address */
 644         return 0;
 645     }
 646     if (!get_pte(env, *vaddr, direntry_size, &entry)) {
 647         return 0;
 648     }
 649
 650     if ((entry & (1 << psn)) && hugepg) {
 651         *huge_page = true;
 652         *hgpg_directory_hit = true;
 653         entry = get_tlb_entry_layout(env, entry, leafentry_size, pf_ptew);
 654         w = directory_index - 1;
 655         if (directory_index & 0x1) {
 656             /* Generate adjacent page from same PTE for odd TLB page */
 657             lsb = BIT_ULL(w) >> 6;
 658             *pw_entrylo0 = entry & ~lsb; /* even page */
 659             *pw_entrylo1 = entry | lsb; /* odd page */
 660         } else if (dph) {
 661             int oddpagebit = 1 << leaf_shift;
 662             uint64_t vaddr2 = *vaddr ^ oddpagebit;
 663             if (*vaddr & oddpagebit) {
 664                 *pw_entrylo1 = entry;
 665             } else {
 666                 *pw_entrylo0 = entry;
 667             }
 668             if (get_physical_address(env, &paddr, &prot, vaddr2, MMU_DATA_LOAD,
 669                                      ptw_mmu_idx) != TLBRET_MATCH) {
 670                 return 0;
 671             }
 672             if (!get_pte(env, vaddr2, leafentry_size, &entry)) {
 673                 return 0;
 674             }
 675             entry = get_tlb_entry_layout(env, entry, leafentry_size, pf_ptew);
 676             if (*vaddr & oddpagebit) {
 677                 *pw_entrylo0 = entry;
 678             } else {
 679                 *pw_entrylo1 = entry;
 680             }
 681         } else {
 682             return 0;
 683         }
 684         return 1;
 685     } else {
 686         *vaddr = entry;
 687         return 2;
 688     }
 689 }
 690
 691 static bool page_table_walk_refill(CPUMIPSState *env, vaddr address,
 692                                    int ptw_mmu_idx)
 693 {
 694     int gdw = (env->CP0_PWSize >> CP0PS_GDW) & 0x3F;
 695     int udw = (env->CP0_PWSize >> CP0PS_UDW) & 0x3F;
 696     int mdw = (env->CP0_PWSize >> CP0PS_MDW) & 0x3F;
 697     int ptw = (env->CP0_PWSize >> CP0PS_PTW) & 0x3F;
 698     int ptew = (env->CP0_PWSize >> CP0PS_PTEW) & 0x3F;
 699
 700     /* Initial values */
 701     bool huge_page = false;
 702     bool hgpg_bdhit = false;
 703     bool hgpg_gdhit = false;
 704     bool hgpg_udhit = false;
 705     bool hgpg_mdhit = false;
 706
 707     int32_t pw_pagemask = 0;
 708     target_ulong pw_entryhi = 0;
 709     uint64_t pw_entrylo0 = 0;
 710     uint64_t pw_entrylo1 = 0;
 711
 712     /* Native pointer size */
 713     /*For the 32-bit architectures, this bit is fixed to 0.*/
 714     int native_shift = (((env->CP0_PWSize >> CP0PS_PS) & 1) == 0) ? 2 : 3;
 715
 716     /* Indices from PWField */
 717     int pf_gdw = (env->CP0_PWField >> CP0PF_GDW) & 0x3F;
 718     int pf_udw = (env->CP0_PWField >> CP0PF_UDW) & 0x3F;
 719     int pf_mdw = (env->CP0_PWField >> CP0PF_MDW) & 0x3F;
 720     int pf_ptw = (env->CP0_PWField >> CP0PF_PTW) & 0x3F;
 721     int pf_ptew = (env->CP0_PWField >> CP0PF_PTEW) & 0x3F;
 722
 723     /* Indices computed from faulting address */
 724     int gindex = (address >> pf_gdw) & ((1 << gdw) - 1);
 725     int uindex = (address >> pf_udw) & ((1 << udw) - 1);
 726     int mindex = (address >> pf_mdw) & ((1 << mdw) - 1);
 727     int ptindex = (address >> pf_ptw) & ((1 << ptw) - 1);
 728
 729     /* Other HTW configs */
 730     int hugepg = (env->CP0_PWCtl >> CP0PC_HUGEPG) & 0x1;
 731     unsigned directory_shift, leaf_shift;
 732
 733     /* Offsets into tables */
 734     unsigned goffset, uoffset, moffset, ptoffset0, ptoffset1;
 735     uint32_t leafentry_size;
 736
 737     /* Starting address - Page Table Base */
 738     uint64_t vaddr = env->CP0_PWBase;
 739
 740     uint64_t dir_entry;
 741     uint64_t paddr;
 742     int prot;
 743     int m;
 744
 745     if (!(env->CP0_Config3 & (1 << CP0C3_PW))) {
 746         /* walker is unimplemented */
 747         return false;
 748     }
 749     if (!(env->CP0_PWCtl & (1 << CP0PC_PWEN))) {
 750         /* walker is disabled */
 751         return false;
 752     }
 753     if (!(gdw > 0 || udw > 0 || mdw > 0)) {
 754         /* no structure to walk */
 755         return false;
 756     }
 757     if (ptew > 1) {
 758         return false;
 759     }
 760
 761     /* HTW Shift values (depend on entry size) */
 762     directory_shift = (hugepg && (ptew == 1)) ? native_shift + 1 : native_shift;
 763     leaf_shift = (ptew == 1) ? native_shift + 1 : native_shift;
 764
 765     goffset = gindex << directory_shift;
 766     uoffset = uindex << directory_shift;
 767     moffset = mindex << directory_shift;
 768     ptoffset0 = (ptindex >> 1) << (leaf_shift + 1);
 769     ptoffset1 = ptoffset0 | (1 << (leaf_shift));
 770
 771     leafentry_size = 1 << (leaf_shift + 3);
 772
 773     /* Global Directory */
 774     if (gdw > 0) {
 775         vaddr |= goffset;
 776         switch (walk_directory(env, &vaddr, pf_gdw, &huge_page, &hgpg_gdhit,
 777                                &pw_entrylo0, &pw_entrylo1,
 778                                directory_shift, leaf_shift, ptw_mmu_idx))
 779         {
 780         case 0:
 781             return false;
 782         case 1:
 783             goto refill;
 784         case 2:
 785         default:
 786             break;
 787         }
 788     }
 789
 790     /* Upper directory */
 791     if (udw > 0) {
 792         vaddr |= uoffset;
 793         switch (walk_directory(env, &vaddr, pf_udw, &huge_page, &hgpg_udhit,
 794                                &pw_entrylo0, &pw_entrylo1,
 795                                directory_shift, leaf_shift, ptw_mmu_idx))
 796         {
 797         case 0:
 798             return false;
 799         case 1:
 800             goto refill;
 801         case 2:
 802         default:
 803             break;
 804         }
 805     }
 806
 807     /* Middle directory */
 808     if (mdw > 0) {
 809         vaddr |= moffset;
 810         switch (walk_directory(env, &vaddr, pf_mdw, &huge_page, &hgpg_mdhit,
 811                                &pw_entrylo0, &pw_entrylo1,
 812                                directory_shift, leaf_shift, ptw_mmu_idx))
 813         {
 814         case 0:
 815             return false;
 816         case 1:
 817             goto refill;
 818         case 2:
 819         default:
 820             break;
 821         }
 822     }
 823
 824     /* Leaf Level Page Table - First half of PTE pair */
 825     vaddr |= ptoffset0;
 826     if (get_physical_address(env, &paddr, &prot, vaddr, MMU_DATA_LOAD,
 827                              ptw_mmu_idx) != TLBRET_MATCH) {
 828         return false;
 829     }
 830     if (!get_pte(env, vaddr, leafentry_size, &dir_entry)) {
 831         return false;
 832     }
 833     dir_entry = get_tlb_entry_layout(env, dir_entry, leafentry_size, pf_ptew);
 834     pw_entrylo0 = dir_entry;
 835
 836     /* Leaf Level Page Table - Second half of PTE pair */
 837     vaddr |= ptoffset1;
 838     if (get_physical_address(env, &paddr, &prot, vaddr, MMU_DATA_LOAD,
 839                              ptw_mmu_idx) != TLBRET_MATCH) {
 840         return false;
 841     }
 842     if (!get_pte(env, vaddr, leafentry_size, &dir_entry)) {
 843         return false;
 844     }
 845     dir_entry = get_tlb_entry_layout(env, dir_entry, leafentry_size, pf_ptew);
 846     pw_entrylo1 = dir_entry;
 847
 848 refill:
 849
 850     m = (1 << pf_ptw) - 1;
 851
 852     if (huge_page) {
 853         switch (hgpg_bdhit << 3 | hgpg_gdhit << 2 | hgpg_udhit << 1 |
 854                 hgpg_mdhit)
 855         {
 856         case 4:
 857             m = (1 << pf_gdw) - 1;
 858             if (pf_gdw & 1) {
 859                 m >>= 1;
 860             }
 861             break;
 862         case 2:
 863             m = (1 << pf_udw) - 1;
 864             if (pf_udw & 1) {
 865                 m >>= 1;
 866             }
 867             break;
 868         case 1:
 869             m = (1 << pf_mdw) - 1;
 870             if (pf_mdw & 1) {
 871                 m >>= 1;
 872             }
 873             break;
 874         }
 875     }
 876     pw_pagemask = m >> TARGET_PAGE_BITS_MIN;
 877     update_pagemask(env, pw_pagemask << CP0PM_MASK, &pw_pagemask);
 878     pw_entryhi = (address & ~0x1fff) | (env->CP0_EntryHi & 0xFF);
 879     {
 880         target_ulong tmp_entryhi = env->CP0_EntryHi;
 881         int32_t tmp_pagemask = env->CP0_PageMask;
 882         uint64_t tmp_entrylo0 = env->CP0_EntryLo0;
 883         uint64_t tmp_entrylo1 = env->CP0_EntryLo1;
 884
 885         env->CP0_EntryHi = pw_entryhi;
 886         env->CP0_PageMask = pw_pagemask;
 887         env->CP0_EntryLo0 = pw_entrylo0;
 888         env->CP0_EntryLo1 = pw_entrylo1;
 889
 890         /*
 891          * The hardware page walker inserts a page into the TLB in a manner
 892          * identical to a TLBWR instruction as executed by the software refill
 893          * handler.
 894          */
 895         r4k_helper_tlbwr(env);
 896
 897         env->CP0_EntryHi = tmp_entryhi;
 898         env->CP0_PageMask = tmp_pagemask;
 899         env->CP0_EntryLo0 = tmp_entrylo0;
 900         env->CP0_EntryLo1 = tmp_entrylo1;
 901     }
 902     return true;
 903 }
 904 #endif
 905
 906 bool mips_cpu_tlb_fill(CPUState *cs, vaddr address, int size,
 907                        MMUAccessType access_type, int mmu_idx,
 908                        bool probe, uintptr_t retaddr)
 909 {
 910     CPUMIPSState *env = cpu_env(cs);
 911     hwaddr physical;
 912     int prot;
 913     int ret = TLBRET_BADADDR;
 914
 915     /* data access */
 916     /* XXX: put correct access by using cpu_restore_state() correctly */
 917     ret = get_physical_address(env, &physical, &prot, address,
 918                                access_type, mmu_idx);
 919     switch (ret) {
 920     case TLBRET_MATCH:
 921         qemu_log_mask(CPU_LOG_MMU,
 922                       "%s address=%" VADDR_PRIx " physical " HWADDR_FMT_plx
 923                       " prot %d\n", __func__, address, physical, prot);
 924         break;
 925     default:
 926         qemu_log_mask(CPU_LOG_MMU,
 927                       "%s address=%" VADDR_PRIx " ret %d\n", __func__, address,
 928                       ret);
 929         break;
 930     }
 931     if (ret == TLBRET_MATCH) {
 932         tlb_set_page(cs, address & TARGET_PAGE_MASK,
 933                      physical & TARGET_PAGE_MASK, prot,
 934                      mmu_idx, TARGET_PAGE_SIZE);
 935         return true;
 936     }
 937 #if !defined(TARGET_MIPS64)
 938     if ((ret == TLBRET_NOMATCH) && (env->tlb->nb_tlb > 1)) {
 939         /*
 940          * Memory reads during hardware page table walking are performed
 941          * as if they were kernel-mode load instructions.
 942          */
 943         int ptw_mmu_idx = (env->hflags & MIPS_HFLAG_ERL ?
 944                            MMU_ERL_IDX : MMU_KERNEL_IDX);
 945
 946         if (page_table_walk_refill(env, address, ptw_mmu_idx)) {
 947             ret = get_physical_address(env, &physical, &prot, address,
 948                                        access_type, mmu_idx);
 949             if (ret == TLBRET_MATCH) {
 950                 tlb_set_page(cs, address & TARGET_PAGE_MASK,
 951                              physical & TARGET_PAGE_MASK, prot,
 952                              mmu_idx, TARGET_PAGE_SIZE);
 953                 return true;
 954             }
 955         }
 956     }
 957 #endif
 958     if (probe) {
 959         return false;
 960     }
 961
 962     raise_mmu_exception(env, address, access_type, ret);
 963     do_raise_exception_err(env, cs->exception_index, env->error_code, retaddr);
 964 }
 965
 966 hwaddr cpu_mips_translate_address(CPUMIPSState *env, target_ulong address,
 967                                   MMUAccessType access_type, uintptr_t retaddr)
 968 {
 969     hwaddr physical;
 970     int prot;
 971     int ret = 0;
 972     CPUState *cs = env_cpu(env);
 973
 974     /* data access */
 975     ret = get_physical_address(env, &physical, &prot, address, access_type,
 976                                mips_env_mmu_index(env));
 977     if (ret == TLBRET_MATCH) {
 978         return physical;
 979     }
 980
 981     raise_mmu_exception(env, address, access_type, ret);
 982     cpu_loop_exit_restore(cs, retaddr);
 983 }
 984
 985 static void set_hflags_for_handler(CPUMIPSState *env)
 986 {
 987     /* Exception handlers are entered in 32-bit mode.  */
 988     env->hflags &= ~(MIPS_HFLAG_M16);
 989     /* ...except that microMIPS lets you choose.  */
 990     if (env->insn_flags & ASE_MICROMIPS) {
 991         env->hflags |= (!!(env->CP0_Config3 &
 992                            (1 << CP0C3_ISA_ON_EXC))
 993                         << MIPS_HFLAG_M16_SHIFT);
 994     }
 995 }
 996
 997 static inline void set_badinstr_registers(CPUMIPSState *env)
 998 {
 999     if (env->insn_flags & ISA_NANOMIPS32) {
1000         if (env->CP0_Config3 & (1 << CP0C3_BI)) {
1001             uint32_t instr = (cpu_lduw_code(env, env->active_tc.PC)) << 16;
1002             if ((instr & 0x10000000) == 0) {
1003                 instr |= cpu_lduw_code(env, env->active_tc.PC + 2);
1004             }
1005             env->CP0_BadInstr = instr;
1006
1007             if ((instr & 0xFC000000) == 0x60000000) {
1008                 instr = cpu_lduw_code(env, env->active_tc.PC + 4) << 16;
1009                 env->CP0_BadInstrX = instr;
1010             }
1011         }
1012         return;
1013     }
1014
1015     if (env->hflags & MIPS_HFLAG_M16) {
1016         /* TODO: add BadInstr support for microMIPS */
1017         return;
1018     }
1019     if (env->CP0_Config3 & (1 << CP0C3_BI)) {
1020         env->CP0_BadInstr = cpu_ldl_code(env, env->active_tc.PC);
1021     }
1022     if ((env->CP0_Config3 & (1 << CP0C3_BP)) &&
1023         (env->hflags & MIPS_HFLAG_BMASK)) {
1024         env->CP0_BadInstrP = cpu_ldl_code(env, env->active_tc.PC - 4);
1025     }
1026 }
1027
1028 void mips_cpu_do_interrupt(CPUState *cs)
1029 {
1030     MIPSCPU *cpu = MIPS_CPU(cs);
1031     CPUMIPSState *env = &cpu->env;
1032     bool update_badinstr = 0;
1033     target_ulong offset;
1034     int cause = -1;
1035
1036     if (qemu_loglevel_mask(CPU_LOG_INT)
1037         && cs->exception_index != EXCP_EXT_INTERRUPT) {
1038         qemu_log("%s enter: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx
1039                  " %s exception\n",
1040                  __func__, env->active_tc.PC, env->CP0_EPC,
1041                  mips_exception_name(cs->exception_index));
1042     }
1043     if (cs->exception_index == EXCP_EXT_INTERRUPT &&
1044         (env->hflags & MIPS_HFLAG_DM)) {
1045         cs->exception_index = EXCP_DINT;
1046     }
1047     offset = 0x180;
1048     switch (cs->exception_index) {
1049     case EXCP_SEMIHOST:
1050         cs->exception_index = EXCP_NONE;
1051         mips_semihosting(env);
1052         env->active_tc.PC += env->error_code;
1053         return;
1054     case EXCP_DSS:
1055         env->CP0_Debug |= 1 << CP0DB_DSS;
1056         /*
1057          * Debug single step cannot be raised inside a delay slot and
1058          * resume will always occur on the next instruction
1059          * (but we assume the pc has always been updated during
1060          * code translation).
1061          */
1062         env->CP0_DEPC = env->active_tc.PC | !!(env->hflags & MIPS_HFLAG_M16);
1063         goto enter_debug_mode;
1064     case EXCP_DINT:
1065         env->CP0_Debug |= 1 << CP0DB_DINT;
1066         goto set_DEPC;
1067     case EXCP_DIB:
1068         env->CP0_Debug |= 1 << CP0DB_DIB;
1069         goto set_DEPC;
1070     case EXCP_DBp:
1071         env->CP0_Debug |= 1 << CP0DB_DBp;
1072         /* Setup DExcCode - SDBBP instruction */
1073         env->CP0_Debug = (env->CP0_Debug & ~(0x1fULL << CP0DB_DEC)) |
1074                          (9 << CP0DB_DEC);
1075         goto set_DEPC;
1076     case EXCP_DDBS:
1077         env->CP0_Debug |= 1 << CP0DB_DDBS;
1078         goto set_DEPC;
1079     case EXCP_DDBL:
1080         env->CP0_Debug |= 1 << CP0DB_DDBL;
1081     set_DEPC:
1082         env->CP0_DEPC = exception_resume_pc(env);
1083         env->hflags &= ~MIPS_HFLAG_BMASK;
1084  enter_debug_mode:
1085         if (env->insn_flags & ISA_MIPS3) {
1086             env->hflags |= MIPS_HFLAG_64;
1087             if (!(env->insn_flags & ISA_MIPS_R6) ||
1088                 env->CP0_Status & (1 << CP0St_KX)) {
1089                 env->hflags &= ~MIPS_HFLAG_AWRAP;
1090             }
1091         }
1092         env->hflags |= MIPS_HFLAG_DM | MIPS_HFLAG_CP0;
1093         env->hflags &= ~(MIPS_HFLAG_KSU);
1094         /* EJTAG probe trap enable is not implemented... */
1095         if (!(env->CP0_Status & (1 << CP0St_EXL))) {
1096             env->CP0_Cause &= ~(1U << CP0Ca_BD);
1097         }
1098         env->active_tc.PC = env->exception_base + 0x480;
1099         set_hflags_for_handler(env);
1100         break;
1101     case EXCP_RESET:
1102         cpu_reset(CPU(cpu));
1103         break;
1104     case EXCP_SRESET:
1105         env->CP0_Status |= (1 << CP0St_SR);
1106         memset(env->CP0_WatchLo, 0, sizeof(env->CP0_WatchLo));
1107         goto set_error_EPC;
1108     case EXCP_NMI:
1109         env->CP0_Status |= (1 << CP0St_NMI);
1110  set_error_EPC:
1111         env->CP0_ErrorEPC = exception_resume_pc(env);
1112         env->hflags &= ~MIPS_HFLAG_BMASK;
1113         env->CP0_Status |= (1 << CP0St_ERL) | (1 << CP0St_BEV);
1114         if (env->insn_flags & ISA_MIPS3) {
1115             env->hflags |= MIPS_HFLAG_64;
1116             if (!(env->insn_flags & ISA_MIPS_R6) ||
1117                 env->CP0_Status & (1 << CP0St_KX)) {
1118                 env->hflags &= ~MIPS_HFLAG_AWRAP;
1119             }
1120         }
1121         env->hflags |= MIPS_HFLAG_CP0;
1122         env->hflags &= ~(MIPS_HFLAG_KSU);
1123         if (!(env->CP0_Status & (1 << CP0St_EXL))) {
1124             env->CP0_Cause &= ~(1U << CP0Ca_BD);
1125         }
1126         env->active_tc.PC = env->exception_base;
1127         set_hflags_for_handler(env);
1128         break;
1129     case EXCP_EXT_INTERRUPT:
1130         cause = 0;
1131         if (env->CP0_Cause & (1 << CP0Ca_IV)) {
1132             uint32_t spacing = (env->CP0_IntCtl >> CP0IntCtl_VS) & 0x1f;
1133
1134             if ((env->CP0_Status & (1 << CP0St_BEV)) || spacing == 0) {
1135                 offset = 0x200;
1136             } else {
1137                 uint32_t vector = 0;
1138                 uint32_t pending = (env->CP0_Cause & CP0Ca_IP_mask) >> CP0Ca_IP;
1139
1140                 if (env->CP0_Config3 & (1 << CP0C3_VEIC)) {
1141                     /*
1142                      * For VEIC mode, the external interrupt controller feeds
1143                      * the vector through the CP0Cause IP lines.
1144                      */
1145                     vector = pending;
1146                 } else {
1147                     /*
1148                      * Vectored Interrupts
1149                      * Mask with Status.IM7-IM0 to get enabled interrupts.
1150                      */
1151                     pending &= (env->CP0_Status >> CP0St_IM) & 0xff;
1152                     /* Find the highest-priority interrupt. */
1153                     while (pending >>= 1) {
1154                         vector++;
1155                     }
1156                 }
1157                 offset = 0x200 + (vector * (spacing << 5));
1158             }
1159         }
1160         goto set_EPC;
1161     case EXCP_LTLBL:
1162         cause = 1;
1163         update_badinstr = !(env->error_code & EXCP_INST_NOTAVAIL);
1164         goto set_EPC;
1165     case EXCP_TLBL:
1166         cause = 2;
1167         update_badinstr = !(env->error_code & EXCP_INST_NOTAVAIL);
1168         if ((env->error_code & EXCP_TLB_NOMATCH) &&
1169             !(env->CP0_Status & (1 << CP0St_EXL))) {
1170 #if defined(TARGET_MIPS64)
1171             int R = env->CP0_BadVAddr >> 62;
1172             int UX = (env->CP0_Status & (1 << CP0St_UX)) != 0;
1173             int KX = (env->CP0_Status & (1 << CP0St_KX)) != 0;
1174
1175             if ((R != 0 || UX) && (R != 3 || KX) &&
1176                 (!(env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)))) {
1177                 offset = 0x080;
1178             } else {
1179 #endif
1180                 offset = 0x000;
1181 #if defined(TARGET_MIPS64)
1182             }
1183 #endif
1184         }
1185         goto set_EPC;
1186     case EXCP_TLBS:
1187         cause = 3;
1188         update_badinstr = 1;
1189         if ((env->error_code & EXCP_TLB_NOMATCH) &&
1190             !(env->CP0_Status & (1 << CP0St_EXL))) {
1191 #if defined(TARGET_MIPS64)
1192             int R = env->CP0_BadVAddr >> 62;
1193             int UX = (env->CP0_Status & (1 << CP0St_UX)) != 0;
1194             int KX = (env->CP0_Status & (1 << CP0St_KX)) != 0;
1195
1196             if ((R != 0 || UX) && (R != 3 || KX) &&
1197                 (!(env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)))) {
1198                 offset = 0x080;
1199             } else {
1200 #endif
1201                 offset = 0x000;
1202 #if defined(TARGET_MIPS64)
1203             }
1204 #endif
1205         }
1206         goto set_EPC;
1207     case EXCP_AdEL:
1208         cause = 4;
1209         update_badinstr = !(env->error_code & EXCP_INST_NOTAVAIL);
1210         goto set_EPC;
1211     case EXCP_AdES:
1212         cause = 5;
1213         update_badinstr = 1;
1214         goto set_EPC;
1215     case EXCP_IBE:
1216         cause = 6;
1217         goto set_EPC;
1218     case EXCP_DBE:
1219         cause = 7;
1220         goto set_EPC;
1221     case EXCP_SYSCALL:
1222         cause = 8;
1223         update_badinstr = 1;
1224         goto set_EPC;
1225     case EXCP_BREAK:
1226         cause = 9;
1227         update_badinstr = 1;
1228         goto set_EPC;
1229     case EXCP_RI:
1230         cause = 10;
1231         update_badinstr = 1;
1232         goto set_EPC;
1233     case EXCP_CpU:
1234         cause = 11;
1235         update_badinstr = 1;
1236         env->CP0_Cause = (env->CP0_Cause & ~(0x3 << CP0Ca_CE)) |
1237                          (env->error_code << CP0Ca_CE);
1238         goto set_EPC;
1239     case EXCP_OVERFLOW:
1240         cause = 12;
1241         update_badinstr = 1;
1242         goto set_EPC;
1243     case EXCP_TRAP:
1244         cause = 13;
1245         update_badinstr = 1;
1246         goto set_EPC;
1247     case EXCP_MSAFPE:
1248         cause = 14;
1249         update_badinstr = 1;
1250         goto set_EPC;
1251     case EXCP_FPE:
1252         cause = 15;
1253         update_badinstr = 1;
1254         goto set_EPC;
1255     case EXCP_C2E:
1256         cause = 18;
1257         goto set_EPC;
1258     case EXCP_TLBRI:
1259         cause = 19;
1260         update_badinstr = 1;
1261         goto set_EPC;
1262     case EXCP_TLBXI:
1263         cause = 20;
1264         goto set_EPC;
1265     case EXCP_MSADIS:
1266         cause = 21;
1267         update_badinstr = 1;
1268         goto set_EPC;
1269     case EXCP_MDMX:
1270         cause = 22;
1271         goto set_EPC;
1272     case EXCP_DWATCH:
1273         cause = 23;
1274         /* XXX: TODO: manage deferred watch exceptions */
1275         goto set_EPC;
1276     case EXCP_MCHECK:
1277         cause = 24;
1278         goto set_EPC;
1279     case EXCP_THREAD:
1280         cause = 25;
1281         goto set_EPC;
1282     case EXCP_DSPDIS:
1283         cause = 26;
1284         goto set_EPC;
1285     case EXCP_CACHE:
1286         cause = 30;
1287         offset = 0x100;
1288  set_EPC:
1289         if (!(env->CP0_Status & (1 << CP0St_EXL))) {
1290             env->CP0_EPC = exception_resume_pc(env);
1291             if (update_badinstr) {
1292                 set_badinstr_registers(env);
1293             }
1294             if (env->hflags & MIPS_HFLAG_BMASK) {
1295                 env->CP0_Cause |= (1U << CP0Ca_BD);
1296             } else {
1297                 env->CP0_Cause &= ~(1U << CP0Ca_BD);
1298             }
1299             env->CP0_Status |= (1 << CP0St_EXL);
1300             if (env->insn_flags & ISA_MIPS3) {
1301                 env->hflags |= MIPS_HFLAG_64;
1302                 if (!(env->insn_flags & ISA_MIPS_R6) ||
1303                     env->CP0_Status & (1 << CP0St_KX)) {
1304                     env->hflags &= ~MIPS_HFLAG_AWRAP;
1305                 }
1306             }
1307             env->hflags |= MIPS_HFLAG_CP0;
1308             env->hflags &= ~(MIPS_HFLAG_KSU);
1309         }
1310         env->hflags &= ~MIPS_HFLAG_BMASK;
1311         if (env->CP0_Status & (1 << CP0St_BEV)) {
1312             env->active_tc.PC = env->exception_base + 0x200;
1313         } else if (cause == 30 && !(env->CP0_Config3 & (1 << CP0C3_SC) &&
1314                                     env->CP0_Config5 & (1 << CP0C5_CV))) {
1315             /* Force KSeg1 for cache errors */
1316             env->active_tc.PC = KSEG1_BASE | (env->CP0_EBase & 0x1FFFF000);
1317         } else {
1318             env->active_tc.PC = env->CP0_EBase & ~0xfff;
1319         }
1320
1321         env->active_tc.PC += offset;
1322         set_hflags_for_handler(env);
1323         env->CP0_Cause = (env->CP0_Cause & ~(0x1f << CP0Ca_EC)) |
1324                          (cause << CP0Ca_EC);
1325         break;
1326     default:
1327         abort();
1328     }
1329     if (qemu_loglevel_mask(CPU_LOG_INT)
1330         && cs->exception_index != EXCP_EXT_INTERRUPT) {
1331         qemu_log("%s: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx " cause %d\n"
1332                  "    S %08x C %08x A " TARGET_FMT_lx " D " TARGET_FMT_lx "\n",
1333                  __func__, env->active_tc.PC, env->CP0_EPC, cause,
1334                  env->CP0_Status, env->CP0_Cause, env->CP0_BadVAddr,
1335                  env->CP0_DEPC);
1336     }
1337     cs->exception_index = EXCP_NONE;
1338 }
1339
1340 bool mips_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
1341 {
1342     if (interrupt_request & CPU_INTERRUPT_HARD) {
1343         CPUMIPSState *env = cpu_env(cs);
1344
1345         if (cpu_mips_hw_interrupts_enabled(env) &&
1346             cpu_mips_hw_interrupts_pending(env)) {
1347             /* Raise it */
1348             cs->exception_index = EXCP_EXT_INTERRUPT;
1349             env->error_code = 0;
1350             mips_cpu_do_interrupt(cs);
1351             return true;
1352         }
1353     }
1354     return false;
1355 }
1356
1357 void r4k_invalidate_tlb(CPUMIPSState *env, int idx, int use_extra)
1358 {
1359     CPUState *cs = env_cpu(env);
1360     r4k_tlb_t *tlb;
1361     target_ulong addr;
1362     target_ulong end;
1363     uint16_t ASID = env->CP0_EntryHi & env->CP0_EntryHi_ASID_mask;
1364     uint32_t MMID = env->CP0_MemoryMapID;
1365     bool mi = !!((env->CP0_Config5 >> CP0C5_MI) & 1);
1366     uint32_t tlb_mmid;
1367     target_ulong mask;
1368
1369     MMID = mi ? MMID : (uint32_t) ASID;
1370
1371     tlb = &env->tlb->mmu.r4k.tlb[idx];
1372     /*
1373      * The qemu TLB is flushed when the ASID/MMID changes, so no need to
1374      * flush these entries again.
1375      */
1376     tlb_mmid = mi ? tlb->MMID : (uint32_t) tlb->ASID;
1377     if (tlb->G == 0 && tlb_mmid != MMID) {
1378         return;
1379     }
1380
1381     if (use_extra && env->tlb->tlb_in_use < MIPS_TLB_MAX) {
1382         /*
1383          * For tlbwr, we can shadow the discarded entry into
1384          * a new (fake) TLB entry, as long as the guest can not
1385          * tell that it's there.
1386          */
1387         env->tlb->mmu.r4k.tlb[env->tlb->tlb_in_use] = *tlb;
1388         env->tlb->tlb_in_use++;
1389         return;
1390     }
1391
1392     /* 1k pages are not supported. */
1393     mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
1394     if (tlb->V0) {
1395         addr = tlb->VPN & ~mask;
1396 #if defined(TARGET_MIPS64)
1397         if (addr >= (0xFFFFFFFF80000000ULL & env->SEGMask)) {
1398             addr |= 0x3FFFFF0000000000ULL;
1399         }
1400 #endif
1401         end = addr | (mask >> 1);
1402         while (addr < end) {
1403             // optimize memset in tlb_flush_page!!!
1404             tlb_flush_page(cs, addr);
1405             addr += TARGET_PAGE_SIZE;
1406         }
1407     }
1408     if (tlb->V1) {
1409         addr = (tlb->VPN & ~mask) | ((mask >> 1) + 1);
1410 #if defined(TARGET_MIPS64)
1411         if (addr >= (0xFFFFFFFF80000000ULL & env->SEGMask)) {
1412             addr |= 0x3FFFFF0000000000ULL;
1413         }
1414 #endif
1415         end = addr | mask;
1416         while (addr - 1 < end) {
1417             // optimize memset in tlb_flush_page!!!
1418             tlb_flush_page(cs, addr);
1419             addr += TARGET_PAGE_SIZE;
1420         }
1421     }
1422 }