target-arm/op_helper.c

   1 /*
   2  *  ARM helper routines
   3  *
   4  *  Copyright (c) 2005-2007 CodeSourcery, LLC
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "qemu/osdep.h"
  20 #include "cpu.h"
  21 #include "exec/helper-proto.h"
  22 #include "internals.h"
  23 #include "exec/exec-all.h"
  24 #include "exec/cpu_ldst.h"
  25
  26 #define SIGNBIT (uint32_t)0x80000000
  27 #define SIGNBIT64 ((uint64_t)1 << 63)
  28
  29 static void raise_exception(CPUARMState *env, uint32_t excp,
  30                             uint32_t syndrome, uint32_t target_el)
  31 {
  32     CPUState *cs = CPU(arm_env_get_cpu(env));
  33
  34     assert(!excp_is_internal(excp));
  35     cs->exception_index = excp;
  36     env->exception.syndrome = syndrome;
  37     env->exception.target_el = target_el;
  38     cpu_loop_exit(cs);
  39 }
  40
  41 static int exception_target_el(CPUARMState *env)
  42 {
  43     int target_el = MAX(1, arm_current_el(env));
  44
  45     /* No such thing as secure EL1 if EL3 is aarch32, so update the target EL
  46      * to EL3 in this case.
  47      */
  48     if (arm_is_secure(env) && !arm_el_is_aa64(env, 3) && target_el == 1) {
  49         target_el = 3;
  50     }
  51
  52     return target_el;
  53 }
  54
  55 uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
  56                           uint32_t rn, uint32_t maxindex)
  57 {
  58     uint32_t val;
  59     uint32_t tmp;
  60     int index;
  61     int shift;
  62     uint64_t *table;
  63     table = (uint64_t *)&env->vfp.regs[rn];
  64     val = 0;
  65     for (shift = 0; shift < 32; shift += 8) {
  66         index = (ireg >> shift) & 0xff;
  67         if (index < maxindex) {
  68             tmp = (table[index >> 3] >> ((index & 7) << 3)) & 0xff;
  69             val |= tmp << shift;
  70         } else {
  71             val |= def & (0xff << shift);
  72         }
  73     }
  74     return val;
  75 }
  76
  77 #if !defined(CONFIG_USER_ONLY)
  78
  79 static inline uint32_t merge_syn_data_abort(uint32_t template_syn,
  80                                             unsigned int target_el,
  81                                             bool same_el,
  82                                             bool s1ptw, int is_write,
  83                                             int fsc)
  84 {
  85     uint32_t syn;
  86
  87     /* ISV is only set for data aborts routed to EL2 and
  88      * never for stage-1 page table walks faulting on stage 2.
  89      *
  90      * Furthermore, ISV is only set for certain kinds of load/stores.
  91      * If the template syndrome does not have ISV set, we should leave
  92      * it cleared.
  93      *
  94      * See ARMv8 specs, D7-1974:
  95      * ISS encoding for an exception from a Data Abort, the
  96      * ISV field.
  97      */
  98     if (!(template_syn & ARM_EL_ISV) || target_el != 2 || s1ptw) {
  99         syn = syn_data_abort_no_iss(same_el,
 100                                     0, 0, s1ptw, is_write == 1, fsc);
 101     } else {
 102         /* Fields: IL, ISV, SAS, SSE, SRT, SF and AR come from the template
 103          * syndrome created at translation time.
 104          * Now we create the runtime syndrome with the remaining fields.
 105          */
 106         syn = syn_data_abort_with_iss(same_el,
 107                                       0, 0, 0, 0, 0,
 108                                       0, 0, s1ptw, is_write == 1, fsc,
 109                                       false);
 110         /* Merge the runtime syndrome with the template syndrome.  */
 111         syn |= template_syn;
 112     }
 113     return syn;
 114 }
 115
 116 /* try to fill the TLB and return an exception if error. If retaddr is
 117  * NULL, it means that the function was called in C code (i.e. not
 118  * from generated code or from helper.c)
 119  */
 120 void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
 121               uintptr_t retaddr)
 122 {
 123     bool ret;
 124     uint32_t fsr = 0;
 125     ARMMMUFaultInfo fi = {};
 126
 127     ret = arm_tlb_fill(cs, addr, is_write, mmu_idx, &fsr, &fi);
 128     if (unlikely(ret)) {
 129         ARMCPU *cpu = ARM_CPU(cs);
 130         CPUARMState *env = &cpu->env;
 131         uint32_t syn, exc;
 132         unsigned int target_el;
 133         bool same_el;
 134
 135         if (retaddr) {
 136             /* now we have a real cpu fault */
 137             cpu_restore_state(cs, retaddr);
 138         }
 139
 140         target_el = exception_target_el(env);
 141         if (fi.stage2) {
 142             target_el = 2;
 143             env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
 144         }
 145         same_el = arm_current_el(env) == target_el;
 146         /* AArch64 syndrome does not have an LPAE bit */
 147         syn = fsr & ~(1 << 9);
 148
 149         /* For insn and data aborts we assume there is no instruction syndrome
 150          * information; this is always true for exceptions reported to EL1.
 151          */
 152         if (is_write == 2) {
 153             syn = syn_insn_abort(same_el, 0, fi.s1ptw, syn);
 154             exc = EXCP_PREFETCH_ABORT;
 155         } else {
 156             syn = merge_syn_data_abort(env->exception.syndrome, target_el,
 157                                        same_el, fi.s1ptw, is_write, syn);
 158             if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
 159                 fsr |= (1 << 11);
 160             }
 161             exc = EXCP_DATA_ABORT;
 162         }
 163
 164         env->exception.vaddress = addr;
 165         env->exception.fsr = fsr;
 166         raise_exception(env, exc, syn, target_el);
 167     }
 168 }
 169
 170 /* Raise a data fault alignment exception for the specified virtual address */
 171 void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, int is_write,
 172                                  int is_user, uintptr_t retaddr)
 173 {
 174     ARMCPU *cpu = ARM_CPU(cs);
 175     CPUARMState *env = &cpu->env;
 176     int target_el;
 177     bool same_el;
 178     uint32_t syn;
 179
 180     if (retaddr) {
 181         /* now we have a real cpu fault */
 182         cpu_restore_state(cs, retaddr);
 183     }
 184
 185     target_el = exception_target_el(env);
 186     same_el = (arm_current_el(env) == target_el);
 187
 188     env->exception.vaddress = vaddr;
 189
 190     /* the DFSR for an alignment fault depends on whether we're using
 191      * the LPAE long descriptor format, or the short descriptor format
 192      */
 193     if (arm_s1_regime_using_lpae_format(env, cpu_mmu_index(env, false))) {
 194         env->exception.fsr = 0x21;
 195     } else {
 196         env->exception.fsr = 0x1;
 197     }
 198
 199     if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
 200         env->exception.fsr |= (1 << 11);
 201     }
 202
 203     syn = merge_syn_data_abort(env->exception.syndrome, target_el,
 204                                same_el, 0, is_write, 0x21);
 205     raise_exception(env, EXCP_DATA_ABORT, syn, target_el);
 206 }
 207
 208 #endif /* !defined(CONFIG_USER_ONLY) */
 209
 210 uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 211 {
 212     uint32_t res = a + b;
 213     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT))
 214         env->QF = 1;
 215     return res;
 216 }
 217
 218 uint32_t HELPER(add_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 219 {
 220     uint32_t res = a + b;
 221     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 222         env->QF = 1;
 223         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 224     }
 225     return res;
 226 }
 227
 228 uint32_t HELPER(sub_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 229 {
 230     uint32_t res = a - b;
 231     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 232         env->QF = 1;
 233         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 234     }
 235     return res;
 236 }
 237
 238 uint32_t HELPER(double_saturate)(CPUARMState *env, int32_t val)
 239 {
 240     uint32_t res;
 241     if (val >= 0x40000000) {
 242         res = ~SIGNBIT;
 243         env->QF = 1;
 244     } else if (val <= (int32_t)0xc0000000) {
 245         res = SIGNBIT;
 246         env->QF = 1;
 247     } else {
 248         res = val << 1;
 249     }
 250     return res;
 251 }
 252
 253 uint32_t HELPER(add_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 254 {
 255     uint32_t res = a + b;
 256     if (res < a) {
 257         env->QF = 1;
 258         res = ~0;
 259     }
 260     return res;
 261 }
 262
 263 uint32_t HELPER(sub_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 264 {
 265     uint32_t res = a - b;
 266     if (res > a) {
 267         env->QF = 1;
 268         res = 0;
 269     }
 270     return res;
 271 }
 272
 273 /* Signed saturation.  */
 274 static inline uint32_t do_ssat(CPUARMState *env, int32_t val, int shift)
 275 {
 276     int32_t top;
 277     uint32_t mask;
 278
 279     top = val >> shift;
 280     mask = (1u << shift) - 1;
 281     if (top > 0) {
 282         env->QF = 1;
 283         return mask;
 284     } else if (top < -1) {
 285         env->QF = 1;
 286         return ~mask;
 287     }
 288     return val;
 289 }
 290
 291 /* Unsigned saturation.  */
 292 static inline uint32_t do_usat(CPUARMState *env, int32_t val, int shift)
 293 {
 294     uint32_t max;
 295
 296     max = (1u << shift) - 1;
 297     if (val < 0) {
 298         env->QF = 1;
 299         return 0;
 300     } else if (val > max) {
 301         env->QF = 1;
 302         return max;
 303     }
 304     return val;
 305 }
 306
 307 /* Signed saturate.  */
 308 uint32_t HELPER(ssat)(CPUARMState *env, uint32_t x, uint32_t shift)
 309 {
 310     return do_ssat(env, x, shift);
 311 }
 312
 313 /* Dual halfword signed saturate.  */
 314 uint32_t HELPER(ssat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 315 {
 316     uint32_t res;
 317
 318     res = (uint16_t)do_ssat(env, (int16_t)x, shift);
 319     res |= do_ssat(env, ((int32_t)x) >> 16, shift) << 16;
 320     return res;
 321 }
 322
 323 /* Unsigned saturate.  */
 324 uint32_t HELPER(usat)(CPUARMState *env, uint32_t x, uint32_t shift)
 325 {
 326     return do_usat(env, x, shift);
 327 }
 328
 329 /* Dual halfword unsigned saturate.  */
 330 uint32_t HELPER(usat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 331 {
 332     uint32_t res;
 333
 334     res = (uint16_t)do_usat(env, (int16_t)x, shift);
 335     res |= do_usat(env, ((int32_t)x) >> 16, shift) << 16;
 336     return res;
 337 }
 338
 339 void HELPER(setend)(CPUARMState *env)
 340 {
 341     env->uncached_cpsr ^= CPSR_E;
 342 }
 343
 344 /* Function checks whether WFx (WFI/WFE) instructions are set up to be trapped.
 345  * The function returns the target EL (1-3) if the instruction is to be trapped;
 346  * otherwise it returns 0 indicating it is not trapped.
 347  */
 348 static inline int check_wfx_trap(CPUARMState *env, bool is_wfe)
 349 {
 350     int cur_el = arm_current_el(env);
 351     uint64_t mask;
 352
 353     /* If we are currently in EL0 then we need to check if SCTLR is set up for
 354      * WFx instructions being trapped to EL1. These trap bits don't exist in v7.
 355      */
 356     if (cur_el < 1 && arm_feature(env, ARM_FEATURE_V8)) {
 357         int target_el;
 358
 359         mask = is_wfe ? SCTLR_nTWE : SCTLR_nTWI;
 360         if (arm_is_secure_below_el3(env) && !arm_el_is_aa64(env, 3)) {
 361             /* Secure EL0 and Secure PL1 is at EL3 */
 362             target_el = 3;
 363         } else {
 364             target_el = 1;
 365         }
 366
 367         if (!(env->cp15.sctlr_el[target_el] & mask)) {
 368             return target_el;
 369         }
 370     }
 371
 372     /* We are not trapping to EL1; trap to EL2 if HCR_EL2 requires it
 373      * No need for ARM_FEATURE check as if HCR_EL2 doesn't exist the
 374      * bits will be zero indicating no trap.
 375      */
 376     if (cur_el < 2 && !arm_is_secure(env)) {
 377         mask = (is_wfe) ? HCR_TWE : HCR_TWI;
 378         if (env->cp15.hcr_el2 & mask) {
 379             return 2;
 380         }
 381     }
 382
 383     /* We are not trapping to EL1 or EL2; trap to EL3 if SCR_EL3 requires it */
 384     if (cur_el < 3) {
 385         mask = (is_wfe) ? SCR_TWE : SCR_TWI;
 386         if (env->cp15.scr_el3 & mask) {
 387             return 3;
 388         }
 389     }
 390
 391     return 0;
 392 }
 393
 394 void HELPER(wfi)(CPUARMState *env)
 395 {
 396     CPUState *cs = CPU(arm_env_get_cpu(env));
 397     int target_el = check_wfx_trap(env, false);
 398
 399     if (cpu_has_work(cs)) {
 400         /* Don't bother to go into our "low power state" if
 401          * we would just wake up immediately.
 402          */
 403         return;
 404     }
 405
 406     if (target_el) {
 407         env->pc -= 4;
 408         raise_exception(env, EXCP_UDEF, syn_wfx(1, 0xe, 0), target_el);
 409     }
 410
 411     cs->exception_index = EXCP_HLT;
 412     cs->halted = 1;
 413     cpu_loop_exit(cs);
 414 }
 415
 416 void HELPER(wfe)(CPUARMState *env)
 417 {
 418     /* This is a hint instruction that is semantically different
 419      * from YIELD even though we currently implement it identically.
 420      * Don't actually halt the CPU, just yield back to top
 421      * level loop. This is not going into a "low power state"
 422      * (ie halting until some event occurs), so we never take
 423      * a configurable trap to a different exception level.
 424      */
 425     HELPER(yield)(env);
 426 }
 427
 428 void HELPER(yield)(CPUARMState *env)
 429 {
 430     ARMCPU *cpu = arm_env_get_cpu(env);
 431     CPUState *cs = CPU(cpu);
 432
 433     /* This is a non-trappable hint instruction that generally indicates
 434      * that the guest is currently busy-looping. Yield control back to the
 435      * top level loop so that a more deserving VCPU has a chance to run.
 436      */
 437     cs->exception_index = EXCP_YIELD;
 438     cpu_loop_exit(cs);
 439 }
 440
 441 /* Raise an internal-to-QEMU exception. This is limited to only
 442  * those EXCP values which are special cases for QEMU to interrupt
 443  * execution and not to be used for exceptions which are passed to
 444  * the guest (those must all have syndrome information and thus should
 445  * use exception_with_syndrome).
 446  */
 447 void HELPER(exception_internal)(CPUARMState *env, uint32_t excp)
 448 {
 449     CPUState *cs = CPU(arm_env_get_cpu(env));
 450
 451     assert(excp_is_internal(excp));
 452     cs->exception_index = excp;
 453     cpu_loop_exit(cs);
 454 }
 455
 456 /* Raise an exception with the specified syndrome register value */
 457 void HELPER(exception_with_syndrome)(CPUARMState *env, uint32_t excp,
 458                                      uint32_t syndrome, uint32_t target_el)
 459 {
 460     raise_exception(env, excp, syndrome, target_el);
 461 }
 462
 463 uint32_t HELPER(cpsr_read)(CPUARMState *env)
 464 {
 465     return cpsr_read(env) & ~(CPSR_EXEC | CPSR_RESERVED);
 466 }
 467
 468 void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 469 {
 470     cpsr_write(env, val, mask, CPSRWriteByInstr);
 471 }
 472
 473 /* Write the CPSR for a 32-bit exception return */
 474 void HELPER(cpsr_write_eret)(CPUARMState *env, uint32_t val)
 475 {
 476     cpsr_write(env, val, CPSR_ERET_MASK, CPSRWriteExceptionReturn);
 477 }
 478
 479 /* Access to user mode registers from privileged modes.  */
 480 uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 481 {
 482     uint32_t val;
 483
 484     if (regno == 13) {
 485         val = env->banked_r13[BANK_USRSYS];
 486     } else if (regno == 14) {
 487         val = env->banked_r14[BANK_USRSYS];
 488     } else if (regno >= 8
 489                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 490         val = env->usr_regs[regno - 8];
 491     } else {
 492         val = env->regs[regno];
 493     }
 494     return val;
 495 }
 496
 497 void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 498 {
 499     if (regno == 13) {
 500         env->banked_r13[BANK_USRSYS] = val;
 501     } else if (regno == 14) {
 502         env->banked_r14[BANK_USRSYS] = val;
 503     } else if (regno >= 8
 504                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 505         env->usr_regs[regno - 8] = val;
 506     } else {
 507         env->regs[regno] = val;
 508     }
 509 }
 510
 511 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
 512 {
 513     if ((env->uncached_cpsr & CPSR_M) == mode) {
 514         env->regs[13] = val;
 515     } else {
 516         env->banked_r13[bank_number(mode)] = val;
 517     }
 518 }
 519
 520 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
 521 {
 522     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_SYS) {
 523         /* SRS instruction is UNPREDICTABLE from System mode; we UNDEF.
 524          * Other UNPREDICTABLE and UNDEF cases were caught at translate time.
 525          */
 526         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 527                         exception_target_el(env));
 528     }
 529
 530     if ((env->uncached_cpsr & CPSR_M) == mode) {
 531         return env->regs[13];
 532     } else {
 533         return env->banked_r13[bank_number(mode)];
 534     }
 535 }
 536
 537 static void msr_mrs_banked_exc_checks(CPUARMState *env, uint32_t tgtmode,
 538                                       uint32_t regno)
 539 {
 540     /* Raise an exception if the requested access is one of the UNPREDICTABLE
 541      * cases; otherwise return. This broadly corresponds to the pseudocode
 542      * BankedRegisterAccessValid() and SPSRAccessValid(),
 543      * except that we have already handled some cases at translate time.
 544      */
 545     int curmode = env->uncached_cpsr & CPSR_M;
 546
 547     if (curmode == tgtmode) {
 548         goto undef;
 549     }
 550
 551     if (tgtmode == ARM_CPU_MODE_USR) {
 552         switch (regno) {
 553         case 8 ... 12:
 554             if (curmode != ARM_CPU_MODE_FIQ) {
 555                 goto undef;
 556             }
 557             break;
 558         case 13:
 559             if (curmode == ARM_CPU_MODE_SYS) {
 560                 goto undef;
 561             }
 562             break;
 563         case 14:
 564             if (curmode == ARM_CPU_MODE_HYP || curmode == ARM_CPU_MODE_SYS) {
 565                 goto undef;
 566             }
 567             break;
 568         default:
 569             break;
 570         }
 571     }
 572
 573     if (tgtmode == ARM_CPU_MODE_HYP) {
 574         switch (regno) {
 575         case 17: /* ELR_Hyp */
 576             if (curmode != ARM_CPU_MODE_HYP && curmode != ARM_CPU_MODE_MON) {
 577                 goto undef;
 578             }
 579             break;
 580         default:
 581             if (curmode != ARM_CPU_MODE_MON) {
 582                 goto undef;
 583             }
 584             break;
 585         }
 586     }
 587
 588     return;
 589
 590 undef:
 591     raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 592                     exception_target_el(env));
 593 }
 594
 595 void HELPER(msr_banked)(CPUARMState *env, uint32_t value, uint32_t tgtmode,
 596                         uint32_t regno)
 597 {
 598     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 599
 600     switch (regno) {
 601     case 16: /* SPSRs */
 602         env->banked_spsr[bank_number(tgtmode)] = value;
 603         break;
 604     case 17: /* ELR_Hyp */
 605         env->elr_el[2] = value;
 606         break;
 607     case 13:
 608         env->banked_r13[bank_number(tgtmode)] = value;
 609         break;
 610     case 14:
 611         env->banked_r14[bank_number(tgtmode)] = value;
 612         break;
 613     case 8 ... 12:
 614         switch (tgtmode) {
 615         case ARM_CPU_MODE_USR:
 616             env->usr_regs[regno - 8] = value;
 617             break;
 618         case ARM_CPU_MODE_FIQ:
 619             env->fiq_regs[regno - 8] = value;
 620             break;
 621         default:
 622             g_assert_not_reached();
 623         }
 624         break;
 625     default:
 626         g_assert_not_reached();
 627     }
 628 }
 629
 630 uint32_t HELPER(mrs_banked)(CPUARMState *env, uint32_t tgtmode, uint32_t regno)
 631 {
 632     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 633
 634     switch (regno) {
 635     case 16: /* SPSRs */
 636         return env->banked_spsr[bank_number(tgtmode)];
 637     case 17: /* ELR_Hyp */
 638         return env->elr_el[2];
 639     case 13:
 640         return env->banked_r13[bank_number(tgtmode)];
 641     case 14:
 642         return env->banked_r14[bank_number(tgtmode)];
 643     case 8 ... 12:
 644         switch (tgtmode) {
 645         case ARM_CPU_MODE_USR:
 646             return env->usr_regs[regno - 8];
 647         case ARM_CPU_MODE_FIQ:
 648             return env->fiq_regs[regno - 8];
 649         default:
 650             g_assert_not_reached();
 651         }
 652     default:
 653         g_assert_not_reached();
 654     }
 655 }
 656
 657 void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome,
 658                                  uint32_t isread)
 659 {
 660     const ARMCPRegInfo *ri = rip;
 661     int target_el;
 662
 663     if (arm_feature(env, ARM_FEATURE_XSCALE) && ri->cp < 14
 664         && extract32(env->cp15.c15_cpar, ri->cp, 1) == 0) {
 665         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 666     }
 667
 668     if (!ri->accessfn) {
 669         return;
 670     }
 671
 672     switch (ri->accessfn(env, ri, isread)) {
 673     case CP_ACCESS_OK:
 674         return;
 675     case CP_ACCESS_TRAP:
 676         target_el = exception_target_el(env);
 677         break;
 678     case CP_ACCESS_TRAP_EL2:
 679         /* Requesting a trap to EL2 when we're in EL3 or S-EL0/1 is
 680          * a bug in the access function.
 681          */
 682         assert(!arm_is_secure(env) && arm_current_el(env) != 3);
 683         target_el = 2;
 684         break;
 685     case CP_ACCESS_TRAP_EL3:
 686         target_el = 3;
 687         break;
 688     case CP_ACCESS_TRAP_UNCATEGORIZED:
 689         target_el = exception_target_el(env);
 690         syndrome = syn_uncategorized();
 691         break;
 692     case CP_ACCESS_TRAP_UNCATEGORIZED_EL2:
 693         target_el = 2;
 694         syndrome = syn_uncategorized();
 695         break;
 696     case CP_ACCESS_TRAP_UNCATEGORIZED_EL3:
 697         target_el = 3;
 698         syndrome = syn_uncategorized();
 699         break;
 700     case CP_ACCESS_TRAP_FP_EL2:
 701         target_el = 2;
 702         /* Since we are an implementation that takes exceptions on a trapped
 703          * conditional insn only if the insn has passed its condition code
 704          * check, we take the IMPDEF choice to always report CV=1 COND=0xe
 705          * (which is also the required value for AArch64 traps).
 706          */
 707         syndrome = syn_fp_access_trap(1, 0xe, false);
 708         break;
 709     case CP_ACCESS_TRAP_FP_EL3:
 710         target_el = 3;
 711         syndrome = syn_fp_access_trap(1, 0xe, false);
 712         break;
 713     default:
 714         g_assert_not_reached();
 715     }
 716
 717     raise_exception(env, EXCP_UDEF, syndrome, target_el);
 718 }
 719
 720 void HELPER(set_cp_reg)(CPUARMState *env, void *rip, uint32_t value)
 721 {
 722     const ARMCPRegInfo *ri = rip;
 723
 724     ri->writefn(env, ri, value);
 725 }
 726
 727 uint32_t HELPER(get_cp_reg)(CPUARMState *env, void *rip)
 728 {
 729     const ARMCPRegInfo *ri = rip;
 730
 731     return ri->readfn(env, ri);
 732 }
 733
 734 void HELPER(set_cp_reg64)(CPUARMState *env, void *rip, uint64_t value)
 735 {
 736     const ARMCPRegInfo *ri = rip;
 737
 738     ri->writefn(env, ri, value);
 739 }
 740
 741 uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
 742 {
 743     const ARMCPRegInfo *ri = rip;
 744
 745     return ri->readfn(env, ri);
 746 }
 747
 748 void HELPER(msr_i_pstate)(CPUARMState *env, uint32_t op, uint32_t imm)
 749 {
 750     /* MSR_i to update PSTATE. This is OK from EL0 only if UMA is set.
 751      * Note that SPSel is never OK from EL0; we rely on handle_msr_i()
 752      * to catch that case at translate time.
 753      */
 754     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
 755         uint32_t syndrome = syn_aa64_sysregtrap(0, extract32(op, 0, 3),
 756                                                 extract32(op, 3, 3), 4,
 757                                                 imm, 0x1f, 0);
 758         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 759     }
 760
 761     switch (op) {
 762     case 0x05: /* SPSel */
 763         update_spsel(env, imm);
 764         break;
 765     case 0x1e: /* DAIFSet */
 766         env->daif |= (imm << 6) & PSTATE_DAIF;
 767         break;
 768     case 0x1f: /* DAIFClear */
 769         env->daif &= ~((imm << 6) & PSTATE_DAIF);
 770         break;
 771     default:
 772         g_assert_not_reached();
 773     }
 774 }
 775
 776 void HELPER(clear_pstate_ss)(CPUARMState *env)
 777 {
 778     env->pstate &= ~PSTATE_SS;
 779 }
 780
 781 void HELPER(pre_hvc)(CPUARMState *env)
 782 {
 783     ARMCPU *cpu = arm_env_get_cpu(env);
 784     int cur_el = arm_current_el(env);
 785     /* FIXME: Use actual secure state.  */
 786     bool secure = false;
 787     bool undef;
 788
 789     if (arm_is_psci_call(cpu, EXCP_HVC)) {
 790         /* If PSCI is enabled and this looks like a valid PSCI call then
 791          * that overrides the architecturally mandated HVC behaviour.
 792          */
 793         return;
 794     }
 795
 796     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 797         /* If EL2 doesn't exist, HVC always UNDEFs */
 798         undef = true;
 799     } else if (arm_feature(env, ARM_FEATURE_EL3)) {
 800         /* EL3.HCE has priority over EL2.HCD. */
 801         undef = !(env->cp15.scr_el3 & SCR_HCE);
 802     } else {
 803         undef = env->cp15.hcr_el2 & HCR_HCD;
 804     }
 805
 806     /* In ARMv7 and ARMv8/AArch32, HVC is undef in secure state.
 807      * For ARMv8/AArch64, HVC is allowed in EL3.
 808      * Note that we've already trapped HVC from EL0 at translation
 809      * time.
 810      */
 811     if (secure && (!is_a64(env) || cur_el == 1)) {
 812         undef = true;
 813     }
 814
 815     if (undef) {
 816         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 817                         exception_target_el(env));
 818     }
 819 }
 820
 821 void HELPER(pre_smc)(CPUARMState *env, uint32_t syndrome)
 822 {
 823     ARMCPU *cpu = arm_env_get_cpu(env);
 824     int cur_el = arm_current_el(env);
 825     bool secure = arm_is_secure(env);
 826     bool smd = env->cp15.scr_el3 & SCR_SMD;
 827     /* On ARMv8 with EL3 AArch64, SMD applies to both S and NS state.
 828      * On ARMv8 with EL3 AArch32, or ARMv7 with the Virtualization
 829      *  extensions, SMD only applies to NS state.
 830      * On ARMv7 without the Virtualization extensions, the SMD bit
 831      * doesn't exist, but we forbid the guest to set it to 1 in scr_write(),
 832      * so we need not special case this here.
 833      */
 834     bool undef = arm_feature(env, ARM_FEATURE_AARCH64) ? smd : smd && !secure;
 835
 836     if (arm_is_psci_call(cpu, EXCP_SMC)) {
 837         /* If PSCI is enabled and this looks like a valid PSCI call then
 838          * that overrides the architecturally mandated SMC behaviour.
 839          */
 840         return;
 841     }
 842
 843     if (!arm_feature(env, ARM_FEATURE_EL3)) {
 844         /* If we have no EL3 then SMC always UNDEFs */
 845         undef = true;
 846     } else if (!secure && cur_el == 1 && (env->cp15.hcr_el2 & HCR_TSC)) {
 847         /* In NS EL1, HCR controlled routing to EL2 has priority over SMD. */
 848         raise_exception(env, EXCP_HYP_TRAP, syndrome, 2);
 849     }
 850
 851     if (undef) {
 852         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 853                         exception_target_el(env));
 854     }
 855 }
 856
 857 static int el_from_spsr(uint32_t spsr)
 858 {
 859     /* Return the exception level that this SPSR is requesting a return to,
 860      * or -1 if it is invalid (an illegal return)
 861      */
 862     if (spsr & PSTATE_nRW) {
 863         switch (spsr & CPSR_M) {
 864         case ARM_CPU_MODE_USR:
 865             return 0;
 866         case ARM_CPU_MODE_HYP:
 867             return 2;
 868         case ARM_CPU_MODE_FIQ:
 869         case ARM_CPU_MODE_IRQ:
 870         case ARM_CPU_MODE_SVC:
 871         case ARM_CPU_MODE_ABT:
 872         case ARM_CPU_MODE_UND:
 873         case ARM_CPU_MODE_SYS:
 874             return 1;
 875         case ARM_CPU_MODE_MON:
 876             /* Returning to Mon from AArch64 is never possible,
 877              * so this is an illegal return.
 878              */
 879         default:
 880             return -1;
 881         }
 882     } else {
 883         if (extract32(spsr, 1, 1)) {
 884             /* Return with reserved M[1] bit set */
 885             return -1;
 886         }
 887         if (extract32(spsr, 0, 4) == 1) {
 888             /* return to EL0 with M[0] bit set */
 889             return -1;
 890         }
 891         return extract32(spsr, 2, 2);
 892     }
 893 }
 894
 895 void HELPER(exception_return)(CPUARMState *env)
 896 {
 897     int cur_el = arm_current_el(env);
 898     unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
 899     uint32_t spsr = env->banked_spsr[spsr_idx];
 900     int new_el;
 901     bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
 902
 903     aarch64_save_sp(env, cur_el);
 904
 905     env->exclusive_addr = -1;
 906
 907     /* We must squash the PSTATE.SS bit to zero unless both of the
 908      * following hold:
 909      *  1. debug exceptions are currently disabled
 910      *  2. singlestep will be active in the EL we return to
 911      * We check 1 here and 2 after we've done the pstate/cpsr write() to
 912      * transition to the EL we're going to.
 913      */
 914     if (arm_generate_debug_exceptions(env)) {
 915         spsr &= ~PSTATE_SS;
 916     }
 917
 918     new_el = el_from_spsr(spsr);
 919     if (new_el == -1) {
 920         goto illegal_return;
 921     }
 922     if (new_el > cur_el
 923         || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
 924         /* Disallow return to an EL which is unimplemented or higher
 925          * than the current one.
 926          */
 927         goto illegal_return;
 928     }
 929
 930     if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
 931         /* Return to an EL which is configured for a different register width */
 932         goto illegal_return;
 933     }
 934
 935     if (new_el == 2 && arm_is_secure_below_el3(env)) {
 936         /* Return to the non-existent secure-EL2 */
 937         goto illegal_return;
 938     }
 939
 940     if (new_el == 1 && (env->cp15.hcr_el2 & HCR_TGE)
 941         && !arm_is_secure_below_el3(env)) {
 942         goto illegal_return;
 943     }
 944
 945     if (!return_to_aa64) {
 946         env->aarch64 = 0;
 947         /* We do a raw CPSR write because aarch64_sync_64_to_32()
 948          * will sort the register banks out for us, and we've already
 949          * caught all the bad-mode cases in el_from_spsr().
 950          */
 951         cpsr_write(env, spsr, ~0, CPSRWriteRaw);
 952         if (!arm_singlestep_active(env)) {
 953             env->uncached_cpsr &= ~PSTATE_SS;
 954         }
 955         aarch64_sync_64_to_32(env);
 956
 957         if (spsr & CPSR_T) {
 958             env->regs[15] = env->elr_el[cur_el] & ~0x1;
 959         } else {
 960             env->regs[15] = env->elr_el[cur_el] & ~0x3;
 961         }
 962     } else {
 963         env->aarch64 = 1;
 964         pstate_write(env, spsr);
 965         if (!arm_singlestep_active(env)) {
 966             env->pstate &= ~PSTATE_SS;
 967         }
 968         aarch64_restore_sp(env, new_el);
 969         env->pc = env->elr_el[cur_el];
 970     }
 971
 972     return;
 973
 974 illegal_return:
 975     /* Illegal return events of various kinds have architecturally
 976      * mandated behaviour:
 977      * restore NZCV and DAIF from SPSR_ELx
 978      * set PSTATE.IL
 979      * restore PC from ELR_ELx
 980      * no change to exception level, execution state or stack pointer
 981      */
 982     env->pstate |= PSTATE_IL;
 983     env->pc = env->elr_el[cur_el];
 984     spsr &= PSTATE_NZCV | PSTATE_DAIF;
 985     spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
 986     pstate_write(env, spsr);
 987     if (!arm_singlestep_active(env)) {
 988         env->pstate &= ~PSTATE_SS;
 989     }
 990 }
 991
 992 /* Return true if the linked breakpoint entry lbn passes its checks */
 993 static bool linked_bp_matches(ARMCPU *cpu, int lbn)
 994 {
 995     CPUARMState *env = &cpu->env;
 996     uint64_t bcr = env->cp15.dbgbcr[lbn];
 997     int brps = extract32(cpu->dbgdidr, 24, 4);
 998     int ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
 999     int bt;
1000     uint32_t contextidr;
1001
1002     /* Links to unimplemented or non-context aware breakpoints are
1003      * CONSTRAINED UNPREDICTABLE: either behave as if disabled, or
1004      * as if linked to an UNKNOWN context-aware breakpoint (in which
1005      * case DBGWCR<n>_EL1.LBN must indicate that breakpoint).
1006      * We choose the former.
1007      */
1008     if (lbn > brps || lbn < (brps - ctx_cmps)) {
1009         return false;
1010     }
1011
1012     bcr = env->cp15.dbgbcr[lbn];
1013
1014     if (extract64(bcr, 0, 1) == 0) {
1015         /* Linked breakpoint disabled : generate no events */
1016         return false;
1017     }
1018
1019     bt = extract64(bcr, 20, 4);
1020
1021     /* We match the whole register even if this is AArch32 using the
1022      * short descriptor format (in which case it holds both PROCID and ASID),
1023      * since we don't implement the optional v7 context ID masking.
1024      */
1025     contextidr = extract64(env->cp15.contextidr_el[1], 0, 32);
1026
1027     switch (bt) {
1028     case 3: /* linked context ID match */
1029         if (arm_current_el(env) > 1) {
1030             /* Context matches never fire in EL2 or (AArch64) EL3 */
1031             return false;
1032         }
1033         return (contextidr == extract64(env->cp15.dbgbvr[lbn], 0, 32));
1034     case 5: /* linked address mismatch (reserved in AArch64) */
1035     case 9: /* linked VMID match (reserved if no EL2) */
1036     case 11: /* linked context ID and VMID match (reserved if no EL2) */
1037     default:
1038         /* Links to Unlinked context breakpoints must generate no
1039          * events; we choose to do the same for reserved values too.
1040          */
1041         return false;
1042     }
1043
1044     return false;
1045 }
1046
1047 static bool bp_wp_matches(ARMCPU *cpu, int n, bool is_wp)
1048 {
1049     CPUARMState *env = &cpu->env;
1050     uint64_t cr;
1051     int pac, hmc, ssc, wt, lbn;
1052     /* Note that for watchpoints the check is against the CPU security
1053      * state, not the S/NS attribute on the offending data access.
1054      */
1055     bool is_secure = arm_is_secure(env);
1056     int access_el = arm_current_el(env);
1057
1058     if (is_wp) {
1059         CPUWatchpoint *wp = env->cpu_watchpoint[n];
1060
1061         if (!wp || !(wp->flags & BP_WATCHPOINT_HIT)) {
1062             return false;
1063         }
1064         cr = env->cp15.dbgwcr[n];
1065         if (wp->hitattrs.user) {
1066             /* The LDRT/STRT/LDT/STT "unprivileged access" instructions should
1067              * match watchpoints as if they were accesses done at EL0, even if
1068              * the CPU is at EL1 or higher.
1069              */
1070             access_el = 0;
1071         }
1072     } else {
1073         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1074
1075         if (!env->cpu_breakpoint[n] || env->cpu_breakpoint[n]->pc != pc) {
1076             return false;
1077         }
1078         cr = env->cp15.dbgbcr[n];
1079     }
1080     /* The WATCHPOINT_HIT flag guarantees us that the watchpoint is
1081      * enabled and that the address and access type match; for breakpoints
1082      * we know the address matched; check the remaining fields, including
1083      * linked breakpoints. We rely on WCR and BCR having the same layout
1084      * for the LBN, SSC, HMC, PAC/PMC and is-linked fields.
1085      * Note that some combinations of {PAC, HMC, SSC} are reserved and
1086      * must act either like some valid combination or as if the watchpoint
1087      * were disabled. We choose the former, and use this together with
1088      * the fact that EL3 must always be Secure and EL2 must always be
1089      * Non-Secure to simplify the code slightly compared to the full
1090      * table in the ARM ARM.
1091      */
1092     pac = extract64(cr, 1, 2);
1093     hmc = extract64(cr, 13, 1);
1094     ssc = extract64(cr, 14, 2);
1095
1096     switch (ssc) {
1097     case 0:
1098         break;
1099     case 1:
1100     case 3:
1101         if (is_secure) {
1102             return false;
1103         }
1104         break;
1105     case 2:
1106         if (!is_secure) {
1107             return false;
1108         }
1109         break;
1110     }
1111
1112     switch (access_el) {
1113     case 3:
1114     case 2:
1115         if (!hmc) {
1116             return false;
1117         }
1118         break;
1119     case 1:
1120         if (extract32(pac, 0, 1) == 0) {
1121             return false;
1122         }
1123         break;
1124     case 0:
1125         if (extract32(pac, 1, 1) == 0) {
1126             return false;
1127         }
1128         break;
1129     default:
1130         g_assert_not_reached();
1131     }
1132
1133     wt = extract64(cr, 20, 1);
1134     lbn = extract64(cr, 16, 4);
1135
1136     if (wt && !linked_bp_matches(cpu, lbn)) {
1137         return false;
1138     }
1139
1140     return true;
1141 }
1142
1143 static bool check_watchpoints(ARMCPU *cpu)
1144 {
1145     CPUARMState *env = &cpu->env;
1146     int n;
1147
1148     /* If watchpoints are disabled globally or we can't take debug
1149      * exceptions here then watchpoint firings are ignored.
1150      */
1151     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1152         || !arm_generate_debug_exceptions(env)) {
1153         return false;
1154     }
1155
1156     for (n = 0; n < ARRAY_SIZE(env->cpu_watchpoint); n++) {
1157         if (bp_wp_matches(cpu, n, true)) {
1158             return true;
1159         }
1160     }
1161     return false;
1162 }
1163
1164 static bool check_breakpoints(ARMCPU *cpu)
1165 {
1166     CPUARMState *env = &cpu->env;
1167     int n;
1168
1169     /* If breakpoints are disabled globally or we can't take debug
1170      * exceptions here then breakpoint firings are ignored.
1171      */
1172     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1173         || !arm_generate_debug_exceptions(env)) {
1174         return false;
1175     }
1176
1177     for (n = 0; n < ARRAY_SIZE(env->cpu_breakpoint); n++) {
1178         if (bp_wp_matches(cpu, n, false)) {
1179             return true;
1180         }
1181     }
1182     return false;
1183 }
1184
1185 void HELPER(check_breakpoints)(CPUARMState *env)
1186 {
1187     ARMCPU *cpu = arm_env_get_cpu(env);
1188
1189     if (check_breakpoints(cpu)) {
1190         HELPER(exception_internal(env, EXCP_DEBUG));
1191     }
1192 }
1193
1194 bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
1195 {
1196     /* Called by core code when a CPU watchpoint fires; need to check if this
1197      * is also an architectural watchpoint match.
1198      */
1199     ARMCPU *cpu = ARM_CPU(cs);
1200
1201     return check_watchpoints(cpu);
1202 }
1203
1204 void arm_debug_excp_handler(CPUState *cs)
1205 {
1206     /* Called by core code when a watchpoint or breakpoint fires;
1207      * need to check which one and raise the appropriate exception.
1208      */
1209     ARMCPU *cpu = ARM_CPU(cs);
1210     CPUARMState *env = &cpu->env;
1211     CPUWatchpoint *wp_hit = cs->watchpoint_hit;
1212
1213     if (wp_hit) {
1214         if (wp_hit->flags & BP_CPU) {
1215             bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
1216             bool same_el = arm_debug_target_el(env) == arm_current_el(env);
1217
1218             cs->watchpoint_hit = NULL;
1219
1220             if (extended_addresses_enabled(env)) {
1221                 env->exception.fsr = (1 << 9) | 0x22;
1222             } else {
1223                 env->exception.fsr = 0x2;
1224             }
1225             env->exception.vaddress = wp_hit->hitaddr;
1226             raise_exception(env, EXCP_DATA_ABORT,
1227                     syn_watchpoint(same_el, 0, wnr),
1228                     arm_debug_target_el(env));
1229         }
1230     } else {
1231         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1232         bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
1233
1234         /* (1) GDB breakpoints should be handled first.
1235          * (2) Do not raise a CPU exception if no CPU breakpoint has fired,
1236          * since singlestep is also done by generating a debug internal
1237          * exception.
1238          */
1239         if (cpu_breakpoint_test(cs, pc, BP_GDB)
1240             || !cpu_breakpoint_test(cs, pc, BP_CPU)) {
1241             return;
1242         }
1243
1244         if (extended_addresses_enabled(env)) {
1245             env->exception.fsr = (1 << 9) | 0x22;
1246         } else {
1247             env->exception.fsr = 0x2;
1248         }
1249         /* FAR is UNKNOWN, so doesn't need setting */
1250         raise_exception(env, EXCP_PREFETCH_ABORT,
1251                         syn_breakpoint(same_el),
1252                         arm_debug_target_el(env));
1253     }
1254 }
1255
1256 /* ??? Flag setting arithmetic is awkward because we need to do comparisons.
1257    The only way to do that in TCG is a conditional branch, which clobbers
1258    all our temporaries.  For now implement these as helper functions.  */
1259
1260 /* Similarly for variable shift instructions.  */
1261
1262 uint32_t HELPER(shl_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1263 {
1264     int shift = i & 0xff;
1265     if (shift >= 32) {
1266         if (shift == 32)
1267             env->CF = x & 1;
1268         else
1269             env->CF = 0;
1270         return 0;
1271     } else if (shift != 0) {
1272         env->CF = (x >> (32 - shift)) & 1;
1273         return x << shift;
1274     }
1275     return x;
1276 }
1277
1278 uint32_t HELPER(shr_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1279 {
1280     int shift = i & 0xff;
1281     if (shift >= 32) {
1282         if (shift == 32)
1283             env->CF = (x >> 31) & 1;
1284         else
1285             env->CF = 0;
1286         return 0;
1287     } else if (shift != 0) {
1288         env->CF = (x >> (shift - 1)) & 1;
1289         return x >> shift;
1290     }
1291     return x;
1292 }
1293
1294 uint32_t HELPER(sar_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1295 {
1296     int shift = i & 0xff;
1297     if (shift >= 32) {
1298         env->CF = (x >> 31) & 1;
1299         return (int32_t)x >> 31;
1300     } else if (shift != 0) {
1301         env->CF = (x >> (shift - 1)) & 1;
1302         return (int32_t)x >> shift;
1303     }
1304     return x;
1305 }
1306
1307 uint32_t HELPER(ror_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1308 {
1309     int shift1, shift;
1310     shift1 = i & 0xff;
1311     shift = shift1 & 0x1f;
1312     if (shift == 0) {
1313         if (shift1 != 0)
1314             env->CF = (x >> 31) & 1;
1315         return x;
1316     } else {
1317         env->CF = (x >> (shift - 1)) & 1;
1318         return ((uint32_t)x >> shift) | (x << (32 - shift));
1319     }
1320 }