target-arm/op_helper.c

   1 /*
   2  *  ARM helper routines
   3  *
   4  *  Copyright (c) 2005-2007 CodeSourcery, LLC
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "qemu/osdep.h"
  20 #include "cpu.h"
  21 #include "exec/helper-proto.h"
  22 #include "internals.h"
  23 #include "exec/exec-all.h"
  24 #include "exec/cpu_ldst.h"
  25
  26 #define SIGNBIT (uint32_t)0x80000000
  27 #define SIGNBIT64 ((uint64_t)1 << 63)
  28
  29 static void raise_exception(CPUARMState *env, uint32_t excp,
  30                             uint32_t syndrome, uint32_t target_el)
  31 {
  32     CPUState *cs = CPU(arm_env_get_cpu(env));
  33
  34     assert(!excp_is_internal(excp));
  35     cs->exception_index = excp;
  36     env->exception.syndrome = syndrome;
  37     env->exception.target_el = target_el;
  38     cpu_loop_exit(cs);
  39 }
  40
  41 static int exception_target_el(CPUARMState *env)
  42 {
  43     int target_el = MAX(1, arm_current_el(env));
  44
  45     /* No such thing as secure EL1 if EL3 is aarch32, so update the target EL
  46      * to EL3 in this case.
  47      */
  48     if (arm_is_secure(env) && !arm_el_is_aa64(env, 3) && target_el == 1) {
  49         target_el = 3;
  50     }
  51
  52     return target_el;
  53 }
  54
  55 uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
  56                           uint32_t rn, uint32_t maxindex)
  57 {
  58     uint32_t val;
  59     uint32_t tmp;
  60     int index;
  61     int shift;
  62     uint64_t *table;
  63     table = (uint64_t *)&env->vfp.regs[rn];
  64     val = 0;
  65     for (shift = 0; shift < 32; shift += 8) {
  66         index = (ireg >> shift) & 0xff;
  67         if (index < maxindex) {
  68             tmp = (table[index >> 3] >> ((index & 7) << 3)) & 0xff;
  69             val |= tmp << shift;
  70         } else {
  71             val |= def & (0xff << shift);
  72         }
  73     }
  74     return val;
  75 }
  76
  77 #if !defined(CONFIG_USER_ONLY)
  78
  79 static inline uint32_t merge_syn_data_abort(uint32_t template_syn,
  80                                             unsigned int target_el,
  81                                             bool same_el,
  82                                             bool s1ptw, int is_write,
  83                                             int fsc)
  84 {
  85     uint32_t syn;
  86
  87     /* ISV is only set for data aborts routed to EL2 and
  88      * never for stage-1 page table walks faulting on stage 2.
  89      *
  90      * Furthermore, ISV is only set for certain kinds of load/stores.
  91      * If the template syndrome does not have ISV set, we should leave
  92      * it cleared.
  93      *
  94      * See ARMv8 specs, D7-1974:
  95      * ISS encoding for an exception from a Data Abort, the
  96      * ISV field.
  97      */
  98     if (!(template_syn & ARM_EL_ISV) || target_el != 2 || s1ptw) {
  99         syn = syn_data_abort_no_iss(same_el,
 100                                     0, 0, s1ptw, is_write == 1, fsc);
 101     } else {
 102         /* Fields: IL, ISV, SAS, SSE, SRT, SF and AR come from the template
 103          * syndrome created at translation time.
 104          * Now we create the runtime syndrome with the remaining fields.
 105          */
 106         syn = syn_data_abort_with_iss(same_el,
 107                                       0, 0, 0, 0, 0,
 108                                       0, 0, s1ptw, is_write == 1, fsc,
 109                                       false);
 110         /* Merge the runtime syndrome with the template syndrome.  */
 111         syn |= template_syn;
 112     }
 113     return syn;
 114 }
 115
 116 /* try to fill the TLB and return an exception if error. If retaddr is
 117  * NULL, it means that the function was called in C code (i.e. not
 118  * from generated code or from helper.c)
 119  */
 120 void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
 121               uintptr_t retaddr)
 122 {
 123     bool ret;
 124     uint32_t fsr = 0;
 125     ARMMMUFaultInfo fi = {};
 126
 127     ret = arm_tlb_fill(cs, addr, is_write, mmu_idx, &fsr, &fi);
 128     if (unlikely(ret)) {
 129         ARMCPU *cpu = ARM_CPU(cs);
 130         CPUARMState *env = &cpu->env;
 131         uint32_t syn, exc;
 132         unsigned int target_el;
 133         bool same_el;
 134
 135         if (retaddr) {
 136             /* now we have a real cpu fault */
 137             cpu_restore_state(cs, retaddr);
 138         }
 139
 140         target_el = exception_target_el(env);
 141         if (fi.stage2) {
 142             target_el = 2;
 143             env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
 144         }
 145         same_el = arm_current_el(env) == target_el;
 146         /* AArch64 syndrome does not have an LPAE bit */
 147         syn = fsr & ~(1 << 9);
 148
 149         /* For insn and data aborts we assume there is no instruction syndrome
 150          * information; this is always true for exceptions reported to EL1.
 151          */
 152         if (is_write == 2) {
 153             syn = syn_insn_abort(same_el, 0, fi.s1ptw, syn);
 154             exc = EXCP_PREFETCH_ABORT;
 155         } else {
 156             syn = merge_syn_data_abort(env->exception.syndrome, target_el,
 157                                        same_el, fi.s1ptw, is_write, syn);
 158             if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
 159                 fsr |= (1 << 11);
 160             }
 161             exc = EXCP_DATA_ABORT;
 162         }
 163
 164         env->exception.vaddress = addr;
 165         env->exception.fsr = fsr;
 166         raise_exception(env, exc, syn, target_el);
 167     }
 168 }
 169
 170 /* Raise a data fault alignment exception for the specified virtual address */
 171 void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, int is_write,
 172                                  int is_user, uintptr_t retaddr)
 173 {
 174     ARMCPU *cpu = ARM_CPU(cs);
 175     CPUARMState *env = &cpu->env;
 176     int target_el;
 177     bool same_el;
 178     uint32_t syn;
 179
 180     if (retaddr) {
 181         /* now we have a real cpu fault */
 182         cpu_restore_state(cs, retaddr);
 183     }
 184
 185     target_el = exception_target_el(env);
 186     same_el = (arm_current_el(env) == target_el);
 187
 188     env->exception.vaddress = vaddr;
 189
 190     /* the DFSR for an alignment fault depends on whether we're using
 191      * the LPAE long descriptor format, or the short descriptor format
 192      */
 193     if (arm_s1_regime_using_lpae_format(env, cpu_mmu_index(env, false))) {
 194         env->exception.fsr = 0x21;
 195     } else {
 196         env->exception.fsr = 0x1;
 197     }
 198
 199     if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
 200         env->exception.fsr |= (1 << 11);
 201     }
 202
 203     syn = merge_syn_data_abort(env->exception.syndrome, target_el,
 204                                same_el, 0, is_write, 0x21);
 205     raise_exception(env, EXCP_DATA_ABORT, syn, target_el);
 206 }
 207
 208 #endif /* !defined(CONFIG_USER_ONLY) */
 209
 210 uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 211 {
 212     uint32_t res = a + b;
 213     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT))
 214         env->QF = 1;
 215     return res;
 216 }
 217
 218 uint32_t HELPER(add_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 219 {
 220     uint32_t res = a + b;
 221     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 222         env->QF = 1;
 223         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 224     }
 225     return res;
 226 }
 227
 228 uint32_t HELPER(sub_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 229 {
 230     uint32_t res = a - b;
 231     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 232         env->QF = 1;
 233         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 234     }
 235     return res;
 236 }
 237
 238 uint32_t HELPER(double_saturate)(CPUARMState *env, int32_t val)
 239 {
 240     uint32_t res;
 241     if (val >= 0x40000000) {
 242         res = ~SIGNBIT;
 243         env->QF = 1;
 244     } else if (val <= (int32_t)0xc0000000) {
 245         res = SIGNBIT;
 246         env->QF = 1;
 247     } else {
 248         res = val << 1;
 249     }
 250     return res;
 251 }
 252
 253 uint32_t HELPER(add_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 254 {
 255     uint32_t res = a + b;
 256     if (res < a) {
 257         env->QF = 1;
 258         res = ~0;
 259     }
 260     return res;
 261 }
 262
 263 uint32_t HELPER(sub_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 264 {
 265     uint32_t res = a - b;
 266     if (res > a) {
 267         env->QF = 1;
 268         res = 0;
 269     }
 270     return res;
 271 }
 272
 273 /* Signed saturation.  */
 274 static inline uint32_t do_ssat(CPUARMState *env, int32_t val, int shift)
 275 {
 276     int32_t top;
 277     uint32_t mask;
 278
 279     top = val >> shift;
 280     mask = (1u << shift) - 1;
 281     if (top > 0) {
 282         env->QF = 1;
 283         return mask;
 284     } else if (top < -1) {
 285         env->QF = 1;
 286         return ~mask;
 287     }
 288     return val;
 289 }
 290
 291 /* Unsigned saturation.  */
 292 static inline uint32_t do_usat(CPUARMState *env, int32_t val, int shift)
 293 {
 294     uint32_t max;
 295
 296     max = (1u << shift) - 1;
 297     if (val < 0) {
 298         env->QF = 1;
 299         return 0;
 300     } else if (val > max) {
 301         env->QF = 1;
 302         return max;
 303     }
 304     return val;
 305 }
 306
 307 /* Signed saturate.  */
 308 uint32_t HELPER(ssat)(CPUARMState *env, uint32_t x, uint32_t shift)
 309 {
 310     return do_ssat(env, x, shift);
 311 }
 312
 313 /* Dual halfword signed saturate.  */
 314 uint32_t HELPER(ssat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 315 {
 316     uint32_t res;
 317
 318     res = (uint16_t)do_ssat(env, (int16_t)x, shift);
 319     res |= do_ssat(env, ((int32_t)x) >> 16, shift) << 16;
 320     return res;
 321 }
 322
 323 /* Unsigned saturate.  */
 324 uint32_t HELPER(usat)(CPUARMState *env, uint32_t x, uint32_t shift)
 325 {
 326     return do_usat(env, x, shift);
 327 }
 328
 329 /* Dual halfword unsigned saturate.  */
 330 uint32_t HELPER(usat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 331 {
 332     uint32_t res;
 333
 334     res = (uint16_t)do_usat(env, (int16_t)x, shift);
 335     res |= do_usat(env, ((int32_t)x) >> 16, shift) << 16;
 336     return res;
 337 }
 338
 339 void HELPER(setend)(CPUARMState *env)
 340 {
 341     env->uncached_cpsr ^= CPSR_E;
 342 }
 343
 344 /* Function checks whether WFx (WFI/WFE) instructions are set up to be trapped.
 345  * The function returns the target EL (1-3) if the instruction is to be trapped;
 346  * otherwise it returns 0 indicating it is not trapped.
 347  */
 348 static inline int check_wfx_trap(CPUARMState *env, bool is_wfe)
 349 {
 350     int cur_el = arm_current_el(env);
 351     uint64_t mask;
 352
 353     /* If we are currently in EL0 then we need to check if SCTLR is set up for
 354      * WFx instructions being trapped to EL1. These trap bits don't exist in v7.
 355      */
 356     if (cur_el < 1 && arm_feature(env, ARM_FEATURE_V8)) {
 357         int target_el;
 358
 359         mask = is_wfe ? SCTLR_nTWE : SCTLR_nTWI;
 360         if (arm_is_secure_below_el3(env) && !arm_el_is_aa64(env, 3)) {
 361             /* Secure EL0 and Secure PL1 is at EL3 */
 362             target_el = 3;
 363         } else {
 364             target_el = 1;
 365         }
 366
 367         if (!(env->cp15.sctlr_el[target_el] & mask)) {
 368             return target_el;
 369         }
 370     }
 371
 372     /* We are not trapping to EL1; trap to EL2 if HCR_EL2 requires it
 373      * No need for ARM_FEATURE check as if HCR_EL2 doesn't exist the
 374      * bits will be zero indicating no trap.
 375      */
 376     if (cur_el < 2 && !arm_is_secure(env)) {
 377         mask = (is_wfe) ? HCR_TWE : HCR_TWI;
 378         if (env->cp15.hcr_el2 & mask) {
 379             return 2;
 380         }
 381     }
 382
 383     /* We are not trapping to EL1 or EL2; trap to EL3 if SCR_EL3 requires it */
 384     if (cur_el < 3) {
 385         mask = (is_wfe) ? SCR_TWE : SCR_TWI;
 386         if (env->cp15.scr_el3 & mask) {
 387             return 3;
 388         }
 389     }
 390
 391     return 0;
 392 }
 393
 394 void HELPER(wfi)(CPUARMState *env)
 395 {
 396     CPUState *cs = CPU(arm_env_get_cpu(env));
 397     int target_el = check_wfx_trap(env, false);
 398
 399     if (cpu_has_work(cs)) {
 400         /* Don't bother to go into our "low power state" if
 401          * we would just wake up immediately.
 402          */
 403         return;
 404     }
 405
 406     if (target_el) {
 407         env->pc -= 4;
 408         raise_exception(env, EXCP_UDEF, syn_wfx(1, 0xe, 0), target_el);
 409     }
 410
 411     cs->exception_index = EXCP_HLT;
 412     cs->halted = 1;
 413     cpu_loop_exit(cs);
 414 }
 415
 416 void HELPER(wfe)(CPUARMState *env)
 417 {
 418     /* This is a hint instruction that is semantically different
 419      * from YIELD even though we currently implement it identically.
 420      * Don't actually halt the CPU, just yield back to top
 421      * level loop. This is not going into a "low power state"
 422      * (ie halting until some event occurs), so we never take
 423      * a configurable trap to a different exception level.
 424      */
 425     HELPER(yield)(env);
 426 }
 427
 428 void HELPER(yield)(CPUARMState *env)
 429 {
 430     ARMCPU *cpu = arm_env_get_cpu(env);
 431     CPUState *cs = CPU(cpu);
 432
 433     /* This is a non-trappable hint instruction that generally indicates
 434      * that the guest is currently busy-looping. Yield control back to the
 435      * top level loop so that a more deserving VCPU has a chance to run.
 436      */
 437     cs->exception_index = EXCP_YIELD;
 438     cpu_loop_exit(cs);
 439 }
 440
 441 /* Raise an internal-to-QEMU exception. This is limited to only
 442  * those EXCP values which are special cases for QEMU to interrupt
 443  * execution and not to be used for exceptions which are passed to
 444  * the guest (those must all have syndrome information and thus should
 445  * use exception_with_syndrome).
 446  */
 447 void HELPER(exception_internal)(CPUARMState *env, uint32_t excp)
 448 {
 449     CPUState *cs = CPU(arm_env_get_cpu(env));
 450
 451     assert(excp_is_internal(excp));
 452     cs->exception_index = excp;
 453     cpu_loop_exit(cs);
 454 }
 455
 456 /* Raise an exception with the specified syndrome register value */
 457 void HELPER(exception_with_syndrome)(CPUARMState *env, uint32_t excp,
 458                                      uint32_t syndrome, uint32_t target_el)
 459 {
 460     raise_exception(env, excp, syndrome, target_el);
 461 }
 462
 463 uint32_t HELPER(cpsr_read)(CPUARMState *env)
 464 {
 465     return cpsr_read(env) & ~(CPSR_EXEC | CPSR_RESERVED);
 466 }
 467
 468 void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 469 {
 470     cpsr_write(env, val, mask, CPSRWriteByInstr);
 471 }
 472
 473 /* Write the CPSR for a 32-bit exception return */
 474 void HELPER(cpsr_write_eret)(CPUARMState *env, uint32_t val)
 475 {
 476     cpsr_write(env, val, CPSR_ERET_MASK, CPSRWriteExceptionReturn);
 477
 478     arm_call_el_change_hook(arm_env_get_cpu(env));
 479 }
 480
 481 /* Access to user mode registers from privileged modes.  */
 482 uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 483 {
 484     uint32_t val;
 485
 486     if (regno == 13) {
 487         val = env->banked_r13[BANK_USRSYS];
 488     } else if (regno == 14) {
 489         val = env->banked_r14[BANK_USRSYS];
 490     } else if (regno >= 8
 491                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 492         val = env->usr_regs[regno - 8];
 493     } else {
 494         val = env->regs[regno];
 495     }
 496     return val;
 497 }
 498
 499 void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 500 {
 501     if (regno == 13) {
 502         env->banked_r13[BANK_USRSYS] = val;
 503     } else if (regno == 14) {
 504         env->banked_r14[BANK_USRSYS] = val;
 505     } else if (regno >= 8
 506                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 507         env->usr_regs[regno - 8] = val;
 508     } else {
 509         env->regs[regno] = val;
 510     }
 511 }
 512
 513 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
 514 {
 515     if ((env->uncached_cpsr & CPSR_M) == mode) {
 516         env->regs[13] = val;
 517     } else {
 518         env->banked_r13[bank_number(mode)] = val;
 519     }
 520 }
 521
 522 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
 523 {
 524     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_SYS) {
 525         /* SRS instruction is UNPREDICTABLE from System mode; we UNDEF.
 526          * Other UNPREDICTABLE and UNDEF cases were caught at translate time.
 527          */
 528         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 529                         exception_target_el(env));
 530     }
 531
 532     if ((env->uncached_cpsr & CPSR_M) == mode) {
 533         return env->regs[13];
 534     } else {
 535         return env->banked_r13[bank_number(mode)];
 536     }
 537 }
 538
 539 static void msr_mrs_banked_exc_checks(CPUARMState *env, uint32_t tgtmode,
 540                                       uint32_t regno)
 541 {
 542     /* Raise an exception if the requested access is one of the UNPREDICTABLE
 543      * cases; otherwise return. This broadly corresponds to the pseudocode
 544      * BankedRegisterAccessValid() and SPSRAccessValid(),
 545      * except that we have already handled some cases at translate time.
 546      */
 547     int curmode = env->uncached_cpsr & CPSR_M;
 548
 549     if (curmode == tgtmode) {
 550         goto undef;
 551     }
 552
 553     if (tgtmode == ARM_CPU_MODE_USR) {
 554         switch (regno) {
 555         case 8 ... 12:
 556             if (curmode != ARM_CPU_MODE_FIQ) {
 557                 goto undef;
 558             }
 559             break;
 560         case 13:
 561             if (curmode == ARM_CPU_MODE_SYS) {
 562                 goto undef;
 563             }
 564             break;
 565         case 14:
 566             if (curmode == ARM_CPU_MODE_HYP || curmode == ARM_CPU_MODE_SYS) {
 567                 goto undef;
 568             }
 569             break;
 570         default:
 571             break;
 572         }
 573     }
 574
 575     if (tgtmode == ARM_CPU_MODE_HYP) {
 576         switch (regno) {
 577         case 17: /* ELR_Hyp */
 578             if (curmode != ARM_CPU_MODE_HYP && curmode != ARM_CPU_MODE_MON) {
 579                 goto undef;
 580             }
 581             break;
 582         default:
 583             if (curmode != ARM_CPU_MODE_MON) {
 584                 goto undef;
 585             }
 586             break;
 587         }
 588     }
 589
 590     return;
 591
 592 undef:
 593     raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 594                     exception_target_el(env));
 595 }
 596
 597 void HELPER(msr_banked)(CPUARMState *env, uint32_t value, uint32_t tgtmode,
 598                         uint32_t regno)
 599 {
 600     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 601
 602     switch (regno) {
 603     case 16: /* SPSRs */
 604         env->banked_spsr[bank_number(tgtmode)] = value;
 605         break;
 606     case 17: /* ELR_Hyp */
 607         env->elr_el[2] = value;
 608         break;
 609     case 13:
 610         env->banked_r13[bank_number(tgtmode)] = value;
 611         break;
 612     case 14:
 613         env->banked_r14[bank_number(tgtmode)] = value;
 614         break;
 615     case 8 ... 12:
 616         switch (tgtmode) {
 617         case ARM_CPU_MODE_USR:
 618             env->usr_regs[regno - 8] = value;
 619             break;
 620         case ARM_CPU_MODE_FIQ:
 621             env->fiq_regs[regno - 8] = value;
 622             break;
 623         default:
 624             g_assert_not_reached();
 625         }
 626         break;
 627     default:
 628         g_assert_not_reached();
 629     }
 630 }
 631
 632 uint32_t HELPER(mrs_banked)(CPUARMState *env, uint32_t tgtmode, uint32_t regno)
 633 {
 634     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 635
 636     switch (regno) {
 637     case 16: /* SPSRs */
 638         return env->banked_spsr[bank_number(tgtmode)];
 639     case 17: /* ELR_Hyp */
 640         return env->elr_el[2];
 641     case 13:
 642         return env->banked_r13[bank_number(tgtmode)];
 643     case 14:
 644         return env->banked_r14[bank_number(tgtmode)];
 645     case 8 ... 12:
 646         switch (tgtmode) {
 647         case ARM_CPU_MODE_USR:
 648             return env->usr_regs[regno - 8];
 649         case ARM_CPU_MODE_FIQ:
 650             return env->fiq_regs[regno - 8];
 651         default:
 652             g_assert_not_reached();
 653         }
 654     default:
 655         g_assert_not_reached();
 656     }
 657 }
 658
 659 void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome,
 660                                  uint32_t isread)
 661 {
 662     const ARMCPRegInfo *ri = rip;
 663     int target_el;
 664
 665     if (arm_feature(env, ARM_FEATURE_XSCALE) && ri->cp < 14
 666         && extract32(env->cp15.c15_cpar, ri->cp, 1) == 0) {
 667         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 668     }
 669
 670     if (!ri->accessfn) {
 671         return;
 672     }
 673
 674     switch (ri->accessfn(env, ri, isread)) {
 675     case CP_ACCESS_OK:
 676         return;
 677     case CP_ACCESS_TRAP:
 678         target_el = exception_target_el(env);
 679         break;
 680     case CP_ACCESS_TRAP_EL2:
 681         /* Requesting a trap to EL2 when we're in EL3 or S-EL0/1 is
 682          * a bug in the access function.
 683          */
 684         assert(!arm_is_secure(env) && arm_current_el(env) != 3);
 685         target_el = 2;
 686         break;
 687     case CP_ACCESS_TRAP_EL3:
 688         target_el = 3;
 689         break;
 690     case CP_ACCESS_TRAP_UNCATEGORIZED:
 691         target_el = exception_target_el(env);
 692         syndrome = syn_uncategorized();
 693         break;
 694     case CP_ACCESS_TRAP_UNCATEGORIZED_EL2:
 695         target_el = 2;
 696         syndrome = syn_uncategorized();
 697         break;
 698     case CP_ACCESS_TRAP_UNCATEGORIZED_EL3:
 699         target_el = 3;
 700         syndrome = syn_uncategorized();
 701         break;
 702     case CP_ACCESS_TRAP_FP_EL2:
 703         target_el = 2;
 704         /* Since we are an implementation that takes exceptions on a trapped
 705          * conditional insn only if the insn has passed its condition code
 706          * check, we take the IMPDEF choice to always report CV=1 COND=0xe
 707          * (which is also the required value for AArch64 traps).
 708          */
 709         syndrome = syn_fp_access_trap(1, 0xe, false);
 710         break;
 711     case CP_ACCESS_TRAP_FP_EL3:
 712         target_el = 3;
 713         syndrome = syn_fp_access_trap(1, 0xe, false);
 714         break;
 715     default:
 716         g_assert_not_reached();
 717     }
 718
 719     raise_exception(env, EXCP_UDEF, syndrome, target_el);
 720 }
 721
 722 void HELPER(set_cp_reg)(CPUARMState *env, void *rip, uint32_t value)
 723 {
 724     const ARMCPRegInfo *ri = rip;
 725
 726     ri->writefn(env, ri, value);
 727 }
 728
 729 uint32_t HELPER(get_cp_reg)(CPUARMState *env, void *rip)
 730 {
 731     const ARMCPRegInfo *ri = rip;
 732
 733     return ri->readfn(env, ri);
 734 }
 735
 736 void HELPER(set_cp_reg64)(CPUARMState *env, void *rip, uint64_t value)
 737 {
 738     const ARMCPRegInfo *ri = rip;
 739
 740     ri->writefn(env, ri, value);
 741 }
 742
 743 uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
 744 {
 745     const ARMCPRegInfo *ri = rip;
 746
 747     return ri->readfn(env, ri);
 748 }
 749
 750 void HELPER(msr_i_pstate)(CPUARMState *env, uint32_t op, uint32_t imm)
 751 {
 752     /* MSR_i to update PSTATE. This is OK from EL0 only if UMA is set.
 753      * Note that SPSel is never OK from EL0; we rely on handle_msr_i()
 754      * to catch that case at translate time.
 755      */
 756     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
 757         uint32_t syndrome = syn_aa64_sysregtrap(0, extract32(op, 0, 3),
 758                                                 extract32(op, 3, 3), 4,
 759                                                 imm, 0x1f, 0);
 760         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 761     }
 762
 763     switch (op) {
 764     case 0x05: /* SPSel */
 765         update_spsel(env, imm);
 766         break;
 767     case 0x1e: /* DAIFSet */
 768         env->daif |= (imm << 6) & PSTATE_DAIF;
 769         break;
 770     case 0x1f: /* DAIFClear */
 771         env->daif &= ~((imm << 6) & PSTATE_DAIF);
 772         break;
 773     default:
 774         g_assert_not_reached();
 775     }
 776 }
 777
 778 void HELPER(clear_pstate_ss)(CPUARMState *env)
 779 {
 780     env->pstate &= ~PSTATE_SS;
 781 }
 782
 783 void HELPER(pre_hvc)(CPUARMState *env)
 784 {
 785     ARMCPU *cpu = arm_env_get_cpu(env);
 786     int cur_el = arm_current_el(env);
 787     /* FIXME: Use actual secure state.  */
 788     bool secure = false;
 789     bool undef;
 790
 791     if (arm_is_psci_call(cpu, EXCP_HVC)) {
 792         /* If PSCI is enabled and this looks like a valid PSCI call then
 793          * that overrides the architecturally mandated HVC behaviour.
 794          */
 795         return;
 796     }
 797
 798     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 799         /* If EL2 doesn't exist, HVC always UNDEFs */
 800         undef = true;
 801     } else if (arm_feature(env, ARM_FEATURE_EL3)) {
 802         /* EL3.HCE has priority over EL2.HCD. */
 803         undef = !(env->cp15.scr_el3 & SCR_HCE);
 804     } else {
 805         undef = env->cp15.hcr_el2 & HCR_HCD;
 806     }
 807
 808     /* In ARMv7 and ARMv8/AArch32, HVC is undef in secure state.
 809      * For ARMv8/AArch64, HVC is allowed in EL3.
 810      * Note that we've already trapped HVC from EL0 at translation
 811      * time.
 812      */
 813     if (secure && (!is_a64(env) || cur_el == 1)) {
 814         undef = true;
 815     }
 816
 817     if (undef) {
 818         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 819                         exception_target_el(env));
 820     }
 821 }
 822
 823 void HELPER(pre_smc)(CPUARMState *env, uint32_t syndrome)
 824 {
 825     ARMCPU *cpu = arm_env_get_cpu(env);
 826     int cur_el = arm_current_el(env);
 827     bool secure = arm_is_secure(env);
 828     bool smd = env->cp15.scr_el3 & SCR_SMD;
 829     /* On ARMv8 with EL3 AArch64, SMD applies to both S and NS state.
 830      * On ARMv8 with EL3 AArch32, or ARMv7 with the Virtualization
 831      *  extensions, SMD only applies to NS state.
 832      * On ARMv7 without the Virtualization extensions, the SMD bit
 833      * doesn't exist, but we forbid the guest to set it to 1 in scr_write(),
 834      * so we need not special case this here.
 835      */
 836     bool undef = arm_feature(env, ARM_FEATURE_AARCH64) ? smd : smd && !secure;
 837
 838     if (arm_is_psci_call(cpu, EXCP_SMC)) {
 839         /* If PSCI is enabled and this looks like a valid PSCI call then
 840          * that overrides the architecturally mandated SMC behaviour.
 841          */
 842         return;
 843     }
 844
 845     if (!arm_feature(env, ARM_FEATURE_EL3)) {
 846         /* If we have no EL3 then SMC always UNDEFs */
 847         undef = true;
 848     } else if (!secure && cur_el == 1 && (env->cp15.hcr_el2 & HCR_TSC)) {
 849         /* In NS EL1, HCR controlled routing to EL2 has priority over SMD. */
 850         raise_exception(env, EXCP_HYP_TRAP, syndrome, 2);
 851     }
 852
 853     if (undef) {
 854         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 855                         exception_target_el(env));
 856     }
 857 }
 858
 859 static int el_from_spsr(uint32_t spsr)
 860 {
 861     /* Return the exception level that this SPSR is requesting a return to,
 862      * or -1 if it is invalid (an illegal return)
 863      */
 864     if (spsr & PSTATE_nRW) {
 865         switch (spsr & CPSR_M) {
 866         case ARM_CPU_MODE_USR:
 867             return 0;
 868         case ARM_CPU_MODE_HYP:
 869             return 2;
 870         case ARM_CPU_MODE_FIQ:
 871         case ARM_CPU_MODE_IRQ:
 872         case ARM_CPU_MODE_SVC:
 873         case ARM_CPU_MODE_ABT:
 874         case ARM_CPU_MODE_UND:
 875         case ARM_CPU_MODE_SYS:
 876             return 1;
 877         case ARM_CPU_MODE_MON:
 878             /* Returning to Mon from AArch64 is never possible,
 879              * so this is an illegal return.
 880              */
 881         default:
 882             return -1;
 883         }
 884     } else {
 885         if (extract32(spsr, 1, 1)) {
 886             /* Return with reserved M[1] bit set */
 887             return -1;
 888         }
 889         if (extract32(spsr, 0, 4) == 1) {
 890             /* return to EL0 with M[0] bit set */
 891             return -1;
 892         }
 893         return extract32(spsr, 2, 2);
 894     }
 895 }
 896
 897 void HELPER(exception_return)(CPUARMState *env)
 898 {
 899     int cur_el = arm_current_el(env);
 900     unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
 901     uint32_t spsr = env->banked_spsr[spsr_idx];
 902     int new_el;
 903     bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
 904
 905     aarch64_save_sp(env, cur_el);
 906
 907     env->exclusive_addr = -1;
 908
 909     /* We must squash the PSTATE.SS bit to zero unless both of the
 910      * following hold:
 911      *  1. debug exceptions are currently disabled
 912      *  2. singlestep will be active in the EL we return to
 913      * We check 1 here and 2 after we've done the pstate/cpsr write() to
 914      * transition to the EL we're going to.
 915      */
 916     if (arm_generate_debug_exceptions(env)) {
 917         spsr &= ~PSTATE_SS;
 918     }
 919
 920     new_el = el_from_spsr(spsr);
 921     if (new_el == -1) {
 922         goto illegal_return;
 923     }
 924     if (new_el > cur_el
 925         || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
 926         /* Disallow return to an EL which is unimplemented or higher
 927          * than the current one.
 928          */
 929         goto illegal_return;
 930     }
 931
 932     if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
 933         /* Return to an EL which is configured for a different register width */
 934         goto illegal_return;
 935     }
 936
 937     if (new_el == 2 && arm_is_secure_below_el3(env)) {
 938         /* Return to the non-existent secure-EL2 */
 939         goto illegal_return;
 940     }
 941
 942     if (new_el == 1 && (env->cp15.hcr_el2 & HCR_TGE)
 943         && !arm_is_secure_below_el3(env)) {
 944         goto illegal_return;
 945     }
 946
 947     if (!return_to_aa64) {
 948         env->aarch64 = 0;
 949         /* We do a raw CPSR write because aarch64_sync_64_to_32()
 950          * will sort the register banks out for us, and we've already
 951          * caught all the bad-mode cases in el_from_spsr().
 952          */
 953         cpsr_write(env, spsr, ~0, CPSRWriteRaw);
 954         if (!arm_singlestep_active(env)) {
 955             env->uncached_cpsr &= ~PSTATE_SS;
 956         }
 957         aarch64_sync_64_to_32(env);
 958
 959         if (spsr & CPSR_T) {
 960             env->regs[15] = env->elr_el[cur_el] & ~0x1;
 961         } else {
 962             env->regs[15] = env->elr_el[cur_el] & ~0x3;
 963         }
 964     } else {
 965         env->aarch64 = 1;
 966         pstate_write(env, spsr);
 967         if (!arm_singlestep_active(env)) {
 968             env->pstate &= ~PSTATE_SS;
 969         }
 970         aarch64_restore_sp(env, new_el);
 971         env->pc = env->elr_el[cur_el];
 972     }
 973
 974     arm_call_el_change_hook(arm_env_get_cpu(env));
 975
 976     return;
 977
 978 illegal_return:
 979     /* Illegal return events of various kinds have architecturally
 980      * mandated behaviour:
 981      * restore NZCV and DAIF from SPSR_ELx
 982      * set PSTATE.IL
 983      * restore PC from ELR_ELx
 984      * no change to exception level, execution state or stack pointer
 985      */
 986     env->pstate |= PSTATE_IL;
 987     env->pc = env->elr_el[cur_el];
 988     spsr &= PSTATE_NZCV | PSTATE_DAIF;
 989     spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
 990     pstate_write(env, spsr);
 991     if (!arm_singlestep_active(env)) {
 992         env->pstate &= ~PSTATE_SS;
 993     }
 994 }
 995
 996 /* Return true if the linked breakpoint entry lbn passes its checks */
 997 static bool linked_bp_matches(ARMCPU *cpu, int lbn)
 998 {
 999     CPUARMState *env = &cpu->env;
1000     uint64_t bcr = env->cp15.dbgbcr[lbn];
1001     int brps = extract32(cpu->dbgdidr, 24, 4);
1002     int ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
1003     int bt;
1004     uint32_t contextidr;
1005
1006     /* Links to unimplemented or non-context aware breakpoints are
1007      * CONSTRAINED UNPREDICTABLE: either behave as if disabled, or
1008      * as if linked to an UNKNOWN context-aware breakpoint (in which
1009      * case DBGWCR<n>_EL1.LBN must indicate that breakpoint).
1010      * We choose the former.
1011      */
1012     if (lbn > brps || lbn < (brps - ctx_cmps)) {
1013         return false;
1014     }
1015
1016     bcr = env->cp15.dbgbcr[lbn];
1017
1018     if (extract64(bcr, 0, 1) == 0) {
1019         /* Linked breakpoint disabled : generate no events */
1020         return false;
1021     }
1022
1023     bt = extract64(bcr, 20, 4);
1024
1025     /* We match the whole register even if this is AArch32 using the
1026      * short descriptor format (in which case it holds both PROCID and ASID),
1027      * since we don't implement the optional v7 context ID masking.
1028      */
1029     contextidr = extract64(env->cp15.contextidr_el[1], 0, 32);
1030
1031     switch (bt) {
1032     case 3: /* linked context ID match */
1033         if (arm_current_el(env) > 1) {
1034             /* Context matches never fire in EL2 or (AArch64) EL3 */
1035             return false;
1036         }
1037         return (contextidr == extract64(env->cp15.dbgbvr[lbn], 0, 32));
1038     case 5: /* linked address mismatch (reserved in AArch64) */
1039     case 9: /* linked VMID match (reserved if no EL2) */
1040     case 11: /* linked context ID and VMID match (reserved if no EL2) */
1041     default:
1042         /* Links to Unlinked context breakpoints must generate no
1043          * events; we choose to do the same for reserved values too.
1044          */
1045         return false;
1046     }
1047
1048     return false;
1049 }
1050
1051 static bool bp_wp_matches(ARMCPU *cpu, int n, bool is_wp)
1052 {
1053     CPUARMState *env = &cpu->env;
1054     uint64_t cr;
1055     int pac, hmc, ssc, wt, lbn;
1056     /* Note that for watchpoints the check is against the CPU security
1057      * state, not the S/NS attribute on the offending data access.
1058      */
1059     bool is_secure = arm_is_secure(env);
1060     int access_el = arm_current_el(env);
1061
1062     if (is_wp) {
1063         CPUWatchpoint *wp = env->cpu_watchpoint[n];
1064
1065         if (!wp || !(wp->flags & BP_WATCHPOINT_HIT)) {
1066             return false;
1067         }
1068         cr = env->cp15.dbgwcr[n];
1069         if (wp->hitattrs.user) {
1070             /* The LDRT/STRT/LDT/STT "unprivileged access" instructions should
1071              * match watchpoints as if they were accesses done at EL0, even if
1072              * the CPU is at EL1 or higher.
1073              */
1074             access_el = 0;
1075         }
1076     } else {
1077         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1078
1079         if (!env->cpu_breakpoint[n] || env->cpu_breakpoint[n]->pc != pc) {
1080             return false;
1081         }
1082         cr = env->cp15.dbgbcr[n];
1083     }
1084     /* The WATCHPOINT_HIT flag guarantees us that the watchpoint is
1085      * enabled and that the address and access type match; for breakpoints
1086      * we know the address matched; check the remaining fields, including
1087      * linked breakpoints. We rely on WCR and BCR having the same layout
1088      * for the LBN, SSC, HMC, PAC/PMC and is-linked fields.
1089      * Note that some combinations of {PAC, HMC, SSC} are reserved and
1090      * must act either like some valid combination or as if the watchpoint
1091      * were disabled. We choose the former, and use this together with
1092      * the fact that EL3 must always be Secure and EL2 must always be
1093      * Non-Secure to simplify the code slightly compared to the full
1094      * table in the ARM ARM.
1095      */
1096     pac = extract64(cr, 1, 2);
1097     hmc = extract64(cr, 13, 1);
1098     ssc = extract64(cr, 14, 2);
1099
1100     switch (ssc) {
1101     case 0:
1102         break;
1103     case 1:
1104     case 3:
1105         if (is_secure) {
1106             return false;
1107         }
1108         break;
1109     case 2:
1110         if (!is_secure) {
1111             return false;
1112         }
1113         break;
1114     }
1115
1116     switch (access_el) {
1117     case 3:
1118     case 2:
1119         if (!hmc) {
1120             return false;
1121         }
1122         break;
1123     case 1:
1124         if (extract32(pac, 0, 1) == 0) {
1125             return false;
1126         }
1127         break;
1128     case 0:
1129         if (extract32(pac, 1, 1) == 0) {
1130             return false;
1131         }
1132         break;
1133     default:
1134         g_assert_not_reached();
1135     }
1136
1137     wt = extract64(cr, 20, 1);
1138     lbn = extract64(cr, 16, 4);
1139
1140     if (wt && !linked_bp_matches(cpu, lbn)) {
1141         return false;
1142     }
1143
1144     return true;
1145 }
1146
1147 static bool check_watchpoints(ARMCPU *cpu)
1148 {
1149     CPUARMState *env = &cpu->env;
1150     int n;
1151
1152     /* If watchpoints are disabled globally or we can't take debug
1153      * exceptions here then watchpoint firings are ignored.
1154      */
1155     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1156         || !arm_generate_debug_exceptions(env)) {
1157         return false;
1158     }
1159
1160     for (n = 0; n < ARRAY_SIZE(env->cpu_watchpoint); n++) {
1161         if (bp_wp_matches(cpu, n, true)) {
1162             return true;
1163         }
1164     }
1165     return false;
1166 }
1167
1168 static bool check_breakpoints(ARMCPU *cpu)
1169 {
1170     CPUARMState *env = &cpu->env;
1171     int n;
1172
1173     /* If breakpoints are disabled globally or we can't take debug
1174      * exceptions here then breakpoint firings are ignored.
1175      */
1176     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1177         || !arm_generate_debug_exceptions(env)) {
1178         return false;
1179     }
1180
1181     for (n = 0; n < ARRAY_SIZE(env->cpu_breakpoint); n++) {
1182         if (bp_wp_matches(cpu, n, false)) {
1183             return true;
1184         }
1185     }
1186     return false;
1187 }
1188
1189 void HELPER(check_breakpoints)(CPUARMState *env)
1190 {
1191     ARMCPU *cpu = arm_env_get_cpu(env);
1192
1193     if (check_breakpoints(cpu)) {
1194         HELPER(exception_internal(env, EXCP_DEBUG));
1195     }
1196 }
1197
1198 bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
1199 {
1200     /* Called by core code when a CPU watchpoint fires; need to check if this
1201      * is also an architectural watchpoint match.
1202      */
1203     ARMCPU *cpu = ARM_CPU(cs);
1204
1205     return check_watchpoints(cpu);
1206 }
1207
1208 void arm_debug_excp_handler(CPUState *cs)
1209 {
1210     /* Called by core code when a watchpoint or breakpoint fires;
1211      * need to check which one and raise the appropriate exception.
1212      */
1213     ARMCPU *cpu = ARM_CPU(cs);
1214     CPUARMState *env = &cpu->env;
1215     CPUWatchpoint *wp_hit = cs->watchpoint_hit;
1216
1217     if (wp_hit) {
1218         if (wp_hit->flags & BP_CPU) {
1219             bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
1220             bool same_el = arm_debug_target_el(env) == arm_current_el(env);
1221
1222             cs->watchpoint_hit = NULL;
1223
1224             if (extended_addresses_enabled(env)) {
1225                 env->exception.fsr = (1 << 9) | 0x22;
1226             } else {
1227                 env->exception.fsr = 0x2;
1228             }
1229             env->exception.vaddress = wp_hit->hitaddr;
1230             raise_exception(env, EXCP_DATA_ABORT,
1231                     syn_watchpoint(same_el, 0, wnr),
1232                     arm_debug_target_el(env));
1233         }
1234     } else {
1235         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1236         bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
1237
1238         /* (1) GDB breakpoints should be handled first.
1239          * (2) Do not raise a CPU exception if no CPU breakpoint has fired,
1240          * since singlestep is also done by generating a debug internal
1241          * exception.
1242          */
1243         if (cpu_breakpoint_test(cs, pc, BP_GDB)
1244             || !cpu_breakpoint_test(cs, pc, BP_CPU)) {
1245             return;
1246         }
1247
1248         if (extended_addresses_enabled(env)) {
1249             env->exception.fsr = (1 << 9) | 0x22;
1250         } else {
1251             env->exception.fsr = 0x2;
1252         }
1253         /* FAR is UNKNOWN, so doesn't need setting */
1254         raise_exception(env, EXCP_PREFETCH_ABORT,
1255                         syn_breakpoint(same_el),
1256                         arm_debug_target_el(env));
1257     }
1258 }
1259
1260 /* ??? Flag setting arithmetic is awkward because we need to do comparisons.
1261    The only way to do that in TCG is a conditional branch, which clobbers
1262    all our temporaries.  For now implement these as helper functions.  */
1263
1264 /* Similarly for variable shift instructions.  */
1265
1266 uint32_t HELPER(shl_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1267 {
1268     int shift = i & 0xff;
1269     if (shift >= 32) {
1270         if (shift == 32)
1271             env->CF = x & 1;
1272         else
1273             env->CF = 0;
1274         return 0;
1275     } else if (shift != 0) {
1276         env->CF = (x >> (32 - shift)) & 1;
1277         return x << shift;
1278     }
1279     return x;
1280 }
1281
1282 uint32_t HELPER(shr_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1283 {
1284     int shift = i & 0xff;
1285     if (shift >= 32) {
1286         if (shift == 32)
1287             env->CF = (x >> 31) & 1;
1288         else
1289             env->CF = 0;
1290         return 0;
1291     } else if (shift != 0) {
1292         env->CF = (x >> (shift - 1)) & 1;
1293         return x >> shift;
1294     }
1295     return x;
1296 }
1297
1298 uint32_t HELPER(sar_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1299 {
1300     int shift = i & 0xff;
1301     if (shift >= 32) {
1302         env->CF = (x >> 31) & 1;
1303         return (int32_t)x >> 31;
1304     } else if (shift != 0) {
1305         env->CF = (x >> (shift - 1)) & 1;
1306         return (int32_t)x >> shift;
1307     }
1308     return x;
1309 }
1310
1311 uint32_t HELPER(ror_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1312 {
1313     int shift1, shift;
1314     shift1 = i & 0xff;
1315     shift = shift1 & 0x1f;
1316     if (shift == 0) {
1317         if (shift1 != 0)
1318             env->CF = (x >> 31) & 1;
1319         return x;
1320     } else {
1321         env->CF = (x >> (shift - 1)) & 1;
1322         return ((uint32_t)x >> shift) | (x << (32 - shift));
1323     }
1324 }