target-arm/op_helper.c

   1 /*
   2  *  ARM helper routines
   3  *
   4  *  Copyright (c) 2005-2007 CodeSourcery, LLC
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "qemu/osdep.h"
  20 #include "cpu.h"
  21 #include "exec/helper-proto.h"
  22 #include "internals.h"
  23 #include "exec/cpu_ldst.h"
  24
  25 #define SIGNBIT (uint32_t)0x80000000
  26 #define SIGNBIT64 ((uint64_t)1 << 63)
  27
  28 static void QEMU_NORETURN
  29 raise_exception(CPUARMState *env, uint32_t excp,
  30                 uint32_t syndrome, uint32_t target_el)
  31 {
  32     CPUState *cs = CPU(arm_env_get_cpu(env));
  33
  34     assert(!excp_is_internal(excp));
  35     cs->exception_index = excp;
  36     env->exception.syndrome = syndrome;
  37     env->exception.target_el = target_el;
  38     cpu_loop_exit(cs);
  39 }
  40
  41 static int exception_target_el(CPUARMState *env)
  42 {
  43     int target_el = MAX(1, arm_current_el(env));
  44
  45     /* No such thing as secure EL1 if EL3 is aarch32, so update the target EL
  46      * to EL3 in this case.
  47      */
  48     if (arm_is_secure(env) && !arm_el_is_aa64(env, 3) && target_el == 1) {
  49         target_el = 3;
  50     }
  51
  52     return target_el;
  53 }
  54
  55 uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
  56                           uint32_t rn, uint32_t maxindex)
  57 {
  58     uint32_t val;
  59     uint32_t tmp;
  60     int index;
  61     int shift;
  62     uint64_t *table;
  63     table = (uint64_t *)&env->vfp.regs[rn];
  64     val = 0;
  65     for (shift = 0; shift < 32; shift += 8) {
  66         index = (ireg >> shift) & 0xff;
  67         if (index < maxindex) {
  68             tmp = (table[index >> 3] >> ((index & 7) << 3)) & 0xff;
  69             val |= tmp << shift;
  70         } else {
  71             val |= def & (0xff << shift);
  72         }
  73     }
  74     return val;
  75 }
  76
  77 #if !defined(CONFIG_USER_ONLY)
  78
  79 /* try to fill the TLB and return an exception if error. If retaddr is
  80  * NULL, it means that the function was called in C code (i.e. not
  81  * from generated code or from helper.c)
  82  */
  83 void tlb_fill(CPUState *cs, target_ulong addr, int is_write, int mmu_idx,
  84               uintptr_t retaddr)
  85 {
  86     bool ret;
  87     uint32_t fsr = 0;
  88     ARMMMUFaultInfo fi = {};
  89
  90     ret = arm_tlb_fill(cs, addr, is_write, mmu_idx, &fsr, &fi);
  91     if (unlikely(ret)) {
  92         ARMCPU *cpu = ARM_CPU(cs);
  93         CPUARMState *env = &cpu->env;
  94         uint32_t syn, exc;
  95         unsigned int target_el;
  96         bool same_el;
  97
  98         if (retaddr) {
  99             /* now we have a real cpu fault */
 100             cpu_restore_state(cs, retaddr);
 101         }
 102
 103         target_el = exception_target_el(env);
 104         if (fi.stage2) {
 105             target_el = 2;
 106             env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
 107         }
 108         same_el = arm_current_el(env) == target_el;
 109         /* AArch64 syndrome does not have an LPAE bit */
 110         syn = fsr & ~(1 << 9);
 111
 112         /* For insn and data aborts we assume there is no instruction syndrome
 113          * information; this is always true for exceptions reported to EL1.
 114          */
 115         if (is_write == 2) {
 116             syn = syn_insn_abort(same_el, 0, fi.s1ptw, syn);
 117             exc = EXCP_PREFETCH_ABORT;
 118         } else {
 119             syn = syn_data_abort_no_iss(same_el,
 120                                         0, 0, fi.s1ptw, is_write == 1, syn);
 121             if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
 122                 fsr |= (1 << 11);
 123             }
 124             exc = EXCP_DATA_ABORT;
 125         }
 126
 127         env->exception.vaddress = addr;
 128         env->exception.fsr = fsr;
 129         raise_exception(env, exc, syn, target_el);
 130     }
 131 }
 132
 133 /* Raise a data fault alignment exception for the specified virtual address */
 134 void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr, int is_write,
 135                                  int is_user, uintptr_t retaddr)
 136 {
 137     ARMCPU *cpu = ARM_CPU(cs);
 138     CPUARMState *env = &cpu->env;
 139     int target_el;
 140     bool same_el;
 141
 142     if (retaddr) {
 143         /* now we have a real cpu fault */
 144         cpu_restore_state(cs, retaddr);
 145     }
 146
 147     target_el = exception_target_el(env);
 148     same_el = (arm_current_el(env) == target_el);
 149
 150     env->exception.vaddress = vaddr;
 151
 152     /* the DFSR for an alignment fault depends on whether we're using
 153      * the LPAE long descriptor format, or the short descriptor format
 154      */
 155     if (arm_s1_regime_using_lpae_format(env, cpu_mmu_index(env, false))) {
 156         env->exception.fsr = 0x21;
 157     } else {
 158         env->exception.fsr = 0x1;
 159     }
 160
 161     if (is_write == 1 && arm_feature(env, ARM_FEATURE_V6)) {
 162         env->exception.fsr |= (1 << 11);
 163     }
 164
 165     raise_exception(env, EXCP_DATA_ABORT,
 166                     syn_data_abort_no_iss(same_el,
 167                                           0, 0, 0, is_write == 1, 0x21),
 168                     target_el);
 169 }
 170
 171 #endif /* !defined(CONFIG_USER_ONLY) */
 172
 173 uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 174 {
 175     uint32_t res = a + b;
 176     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT))
 177         env->QF = 1;
 178     return res;
 179 }
 180
 181 uint32_t HELPER(add_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 182 {
 183     uint32_t res = a + b;
 184     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 185         env->QF = 1;
 186         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 187     }
 188     return res;
 189 }
 190
 191 uint32_t HELPER(sub_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 192 {
 193     uint32_t res = a - b;
 194     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 195         env->QF = 1;
 196         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 197     }
 198     return res;
 199 }
 200
 201 uint32_t HELPER(double_saturate)(CPUARMState *env, int32_t val)
 202 {
 203     uint32_t res;
 204     if (val >= 0x40000000) {
 205         res = ~SIGNBIT;
 206         env->QF = 1;
 207     } else if (val <= (int32_t)0xc0000000) {
 208         res = SIGNBIT;
 209         env->QF = 1;
 210     } else {
 211         res = val << 1;
 212     }
 213     return res;
 214 }
 215
 216 uint32_t HELPER(add_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 217 {
 218     uint32_t res = a + b;
 219     if (res < a) {
 220         env->QF = 1;
 221         res = ~0;
 222     }
 223     return res;
 224 }
 225
 226 uint32_t HELPER(sub_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 227 {
 228     uint32_t res = a - b;
 229     if (res > a) {
 230         env->QF = 1;
 231         res = 0;
 232     }
 233     return res;
 234 }
 235
 236 /* Signed saturation.  */
 237 static inline uint32_t do_ssat(CPUARMState *env, int32_t val, int shift)
 238 {
 239     int32_t top;
 240     uint32_t mask;
 241
 242     top = val >> shift;
 243     mask = (1u << shift) - 1;
 244     if (top > 0) {
 245         env->QF = 1;
 246         return mask;
 247     } else if (top < -1) {
 248         env->QF = 1;
 249         return ~mask;
 250     }
 251     return val;
 252 }
 253
 254 /* Unsigned saturation.  */
 255 static inline uint32_t do_usat(CPUARMState *env, int32_t val, int shift)
 256 {
 257     uint32_t max;
 258
 259     max = (1u << shift) - 1;
 260     if (val < 0) {
 261         env->QF = 1;
 262         return 0;
 263     } else if (val > max) {
 264         env->QF = 1;
 265         return max;
 266     }
 267     return val;
 268 }
 269
 270 /* Signed saturate.  */
 271 uint32_t HELPER(ssat)(CPUARMState *env, uint32_t x, uint32_t shift)
 272 {
 273     return do_ssat(env, x, shift);
 274 }
 275
 276 /* Dual halfword signed saturate.  */
 277 uint32_t HELPER(ssat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 278 {
 279     uint32_t res;
 280
 281     res = (uint16_t)do_ssat(env, (int16_t)x, shift);
 282     res |= do_ssat(env, ((int32_t)x) >> 16, shift) << 16;
 283     return res;
 284 }
 285
 286 /* Unsigned saturate.  */
 287 uint32_t HELPER(usat)(CPUARMState *env, uint32_t x, uint32_t shift)
 288 {
 289     return do_usat(env, x, shift);
 290 }
 291
 292 /* Dual halfword unsigned saturate.  */
 293 uint32_t HELPER(usat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 294 {
 295     uint32_t res;
 296
 297     res = (uint16_t)do_usat(env, (int16_t)x, shift);
 298     res |= do_usat(env, ((int32_t)x) >> 16, shift) << 16;
 299     return res;
 300 }
 301
 302 void HELPER(setend)(CPUARMState *env)
 303 {
 304     env->uncached_cpsr ^= CPSR_E;
 305 }
 306
 307 /* Function checks whether WFx (WFI/WFE) instructions are set up to be trapped.
 308  * The function returns the target EL (1-3) if the instruction is to be trapped;
 309  * otherwise it returns 0 indicating it is not trapped.
 310  */
 311 static inline int check_wfx_trap(CPUARMState *env, bool is_wfe)
 312 {
 313     int cur_el = arm_current_el(env);
 314     uint64_t mask;
 315
 316     /* If we are currently in EL0 then we need to check if SCTLR is set up for
 317      * WFx instructions being trapped to EL1. These trap bits don't exist in v7.
 318      */
 319     if (cur_el < 1 && arm_feature(env, ARM_FEATURE_V8)) {
 320         int target_el;
 321
 322         mask = is_wfe ? SCTLR_nTWE : SCTLR_nTWI;
 323         if (arm_is_secure_below_el3(env) && !arm_el_is_aa64(env, 3)) {
 324             /* Secure EL0 and Secure PL1 is at EL3 */
 325             target_el = 3;
 326         } else {
 327             target_el = 1;
 328         }
 329
 330         if (!(env->cp15.sctlr_el[target_el] & mask)) {
 331             return target_el;
 332         }
 333     }
 334
 335     /* We are not trapping to EL1; trap to EL2 if HCR_EL2 requires it
 336      * No need for ARM_FEATURE check as if HCR_EL2 doesn't exist the
 337      * bits will be zero indicating no trap.
 338      */
 339     if (cur_el < 2 && !arm_is_secure(env)) {
 340         mask = (is_wfe) ? HCR_TWE : HCR_TWI;
 341         if (env->cp15.hcr_el2 & mask) {
 342             return 2;
 343         }
 344     }
 345
 346     /* We are not trapping to EL1 or EL2; trap to EL3 if SCR_EL3 requires it */
 347     if (cur_el < 3) {
 348         mask = (is_wfe) ? SCR_TWE : SCR_TWI;
 349         if (env->cp15.scr_el3 & mask) {
 350             return 3;
 351         }
 352     }
 353
 354     return 0;
 355 }
 356
 357 void HELPER(wfi)(CPUARMState *env)
 358 {
 359     CPUState *cs = CPU(arm_env_get_cpu(env));
 360     int target_el = check_wfx_trap(env, false);
 361
 362     if (cpu_has_work(cs)) {
 363         /* Don't bother to go into our "low power state" if
 364          * we would just wake up immediately.
 365          */
 366         return;
 367     }
 368
 369     if (target_el) {
 370         env->pc -= 4;
 371         raise_exception(env, EXCP_UDEF, syn_wfx(1, 0xe, 0), target_el);
 372     }
 373
 374     cs->exception_index = EXCP_HLT;
 375     cs->halted = 1;
 376     cpu_loop_exit(cs);
 377 }
 378
 379 void QEMU_NORETURN HELPER(wfe)(CPUARMState *env)
 380 {
 381     /* This is a hint instruction that is semantically different
 382      * from YIELD even though we currently implement it identically.
 383      * Don't actually halt the CPU, just yield back to top
 384      * level loop. This is not going into a "low power state"
 385      * (ie halting until some event occurs), so we never take
 386      * a configurable trap to a different exception level.
 387      */
 388     HELPER(yield)(env);
 389 }
 390
 391 void QEMU_NORETURN HELPER(yield)(CPUARMState *env)
 392 {
 393     ARMCPU *cpu = arm_env_get_cpu(env);
 394     CPUState *cs = CPU(cpu);
 395
 396     /* This is a non-trappable hint instruction that generally indicates
 397      * that the guest is currently busy-looping. Yield control back to the
 398      * top level loop so that a more deserving VCPU has a chance to run.
 399      */
 400     cs->exception_index = EXCP_YIELD;
 401     cpu_loop_exit(cs);
 402 }
 403
 404 /* Raise an internal-to-QEMU exception. This is limited to only
 405  * those EXCP values which are special cases for QEMU to interrupt
 406  * execution and not to be used for exceptions which are passed to
 407  * the guest (those must all have syndrome information and thus should
 408  * use exception_with_syndrome).
 409  */
 410 void QEMU_NORETURN HELPER(exception_internal)(CPUARMState *env, uint32_t excp)
 411 {
 412     CPUState *cs = CPU(arm_env_get_cpu(env));
 413
 414     assert(excp_is_internal(excp));
 415     cs->exception_index = excp;
 416     cpu_loop_exit(cs);
 417 }
 418
 419 /* Raise an exception with the specified syndrome register value */
 420 void QEMU_NORETURN
 421 HELPER(exception_with_syndrome)(CPUARMState *env, uint32_t excp,
 422                                 uint32_t syndrome, uint32_t target_el)
 423 {
 424     raise_exception(env, excp, syndrome, target_el);
 425 }
 426
 427 uint32_t HELPER(cpsr_read)(CPUARMState *env)
 428 {
 429     return cpsr_read(env) & ~(CPSR_EXEC | CPSR_RESERVED);
 430 }
 431
 432 void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 433 {
 434     cpsr_write(env, val, mask, CPSRWriteByInstr);
 435 }
 436
 437 /* Write the CPSR for a 32-bit exception return */
 438 void HELPER(cpsr_write_eret)(CPUARMState *env, uint32_t val)
 439 {
 440     cpsr_write(env, val, CPSR_ERET_MASK, CPSRWriteExceptionReturn);
 441 }
 442
 443 /* Access to user mode registers from privileged modes.  */
 444 uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 445 {
 446     uint32_t val;
 447
 448     if (regno == 13) {
 449         val = env->banked_r13[BANK_USRSYS];
 450     } else if (regno == 14) {
 451         val = env->banked_r14[BANK_USRSYS];
 452     } else if (regno >= 8
 453                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 454         val = env->usr_regs[regno - 8];
 455     } else {
 456         val = env->regs[regno];
 457     }
 458     return val;
 459 }
 460
 461 void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 462 {
 463     if (regno == 13) {
 464         env->banked_r13[BANK_USRSYS] = val;
 465     } else if (regno == 14) {
 466         env->banked_r14[BANK_USRSYS] = val;
 467     } else if (regno >= 8
 468                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 469         env->usr_regs[regno - 8] = val;
 470     } else {
 471         env->regs[regno] = val;
 472     }
 473 }
 474
 475 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
 476 {
 477     if ((env->uncached_cpsr & CPSR_M) == mode) {
 478         env->regs[13] = val;
 479     } else {
 480         env->banked_r13[bank_number(mode)] = val;
 481     }
 482 }
 483
 484 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
 485 {
 486     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_SYS) {
 487         /* SRS instruction is UNPREDICTABLE from System mode; we UNDEF.
 488          * Other UNPREDICTABLE and UNDEF cases were caught at translate time.
 489          */
 490         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 491                         exception_target_el(env));
 492     }
 493
 494     if ((env->uncached_cpsr & CPSR_M) == mode) {
 495         return env->regs[13];
 496     } else {
 497         return env->banked_r13[bank_number(mode)];
 498     }
 499 }
 500
 501 static void msr_mrs_banked_exc_checks(CPUARMState *env, uint32_t tgtmode,
 502                                       uint32_t regno)
 503 {
 504     /* Raise an exception if the requested access is one of the UNPREDICTABLE
 505      * cases; otherwise return. This broadly corresponds to the pseudocode
 506      * BankedRegisterAccessValid() and SPSRAccessValid(),
 507      * except that we have already handled some cases at translate time.
 508      */
 509     int curmode = env->uncached_cpsr & CPSR_M;
 510
 511     if (curmode == tgtmode) {
 512         goto undef;
 513     }
 514
 515     if (tgtmode == ARM_CPU_MODE_USR) {
 516         switch (regno) {
 517         case 8 ... 12:
 518             if (curmode != ARM_CPU_MODE_FIQ) {
 519                 goto undef;
 520             }
 521             break;
 522         case 13:
 523             if (curmode == ARM_CPU_MODE_SYS) {
 524                 goto undef;
 525             }
 526             break;
 527         case 14:
 528             if (curmode == ARM_CPU_MODE_HYP || curmode == ARM_CPU_MODE_SYS) {
 529                 goto undef;
 530             }
 531             break;
 532         default:
 533             break;
 534         }
 535     }
 536
 537     if (tgtmode == ARM_CPU_MODE_HYP) {
 538         switch (regno) {
 539         case 17: /* ELR_Hyp */
 540             if (curmode != ARM_CPU_MODE_HYP && curmode != ARM_CPU_MODE_MON) {
 541                 goto undef;
 542             }
 543             break;
 544         default:
 545             if (curmode != ARM_CPU_MODE_MON) {
 546                 goto undef;
 547             }
 548             break;
 549         }
 550     }
 551
 552     return;
 553
 554 undef:
 555     raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 556                     exception_target_el(env));
 557 }
 558
 559 void HELPER(msr_banked)(CPUARMState *env, uint32_t value, uint32_t tgtmode,
 560                         uint32_t regno)
 561 {
 562     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 563
 564     switch (regno) {
 565     case 16: /* SPSRs */
 566         env->banked_spsr[bank_number(tgtmode)] = value;
 567         break;
 568     case 17: /* ELR_Hyp */
 569         env->elr_el[2] = value;
 570         break;
 571     case 13:
 572         env->banked_r13[bank_number(tgtmode)] = value;
 573         break;
 574     case 14:
 575         env->banked_r14[bank_number(tgtmode)] = value;
 576         break;
 577     case 8 ... 12:
 578         switch (tgtmode) {
 579         case ARM_CPU_MODE_USR:
 580             env->usr_regs[regno - 8] = value;
 581             break;
 582         case ARM_CPU_MODE_FIQ:
 583             env->fiq_regs[regno - 8] = value;
 584             break;
 585         default:
 586             g_assert_not_reached();
 587         }
 588         break;
 589     default:
 590         g_assert_not_reached();
 591     }
 592 }
 593
 594 uint32_t HELPER(mrs_banked)(CPUARMState *env, uint32_t tgtmode, uint32_t regno)
 595 {
 596     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 597
 598     switch (regno) {
 599     case 16: /* SPSRs */
 600         return env->banked_spsr[bank_number(tgtmode)];
 601     case 17: /* ELR_Hyp */
 602         return env->elr_el[2];
 603     case 13:
 604         return env->banked_r13[bank_number(tgtmode)];
 605     case 14:
 606         return env->banked_r14[bank_number(tgtmode)];
 607     case 8 ... 12:
 608         switch (tgtmode) {
 609         case ARM_CPU_MODE_USR:
 610             return env->usr_regs[regno - 8];
 611         case ARM_CPU_MODE_FIQ:
 612             return env->fiq_regs[regno - 8];
 613         default:
 614             g_assert_not_reached();
 615         }
 616     default:
 617         g_assert_not_reached();
 618     }
 619 }
 620
 621 void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome,
 622                                  uint32_t isread)
 623 {
 624     const ARMCPRegInfo *ri = rip;
 625     int target_el;
 626
 627     if (arm_feature(env, ARM_FEATURE_XSCALE) && ri->cp < 14
 628         && extract32(env->cp15.c15_cpar, ri->cp, 1) == 0) {
 629         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 630     }
 631
 632     if (!ri->accessfn) {
 633         return;
 634     }
 635
 636     switch (ri->accessfn(env, ri, isread)) {
 637     case CP_ACCESS_OK:
 638         return;
 639     case CP_ACCESS_TRAP:
 640         target_el = exception_target_el(env);
 641         break;
 642     case CP_ACCESS_TRAP_EL2:
 643         /* Requesting a trap to EL2 when we're in EL3 or S-EL0/1 is
 644          * a bug in the access function.
 645          */
 646         assert(!arm_is_secure(env) && arm_current_el(env) != 3);
 647         target_el = 2;
 648         break;
 649     case CP_ACCESS_TRAP_EL3:
 650         target_el = 3;
 651         break;
 652     case CP_ACCESS_TRAP_UNCATEGORIZED:
 653         target_el = exception_target_el(env);
 654         syndrome = syn_uncategorized();
 655         break;
 656     case CP_ACCESS_TRAP_UNCATEGORIZED_EL2:
 657         target_el = 2;
 658         syndrome = syn_uncategorized();
 659         break;
 660     case CP_ACCESS_TRAP_UNCATEGORIZED_EL3:
 661         target_el = 3;
 662         syndrome = syn_uncategorized();
 663         break;
 664     case CP_ACCESS_TRAP_FP_EL2:
 665         target_el = 2;
 666         /* Since we are an implementation that takes exceptions on a trapped
 667          * conditional insn only if the insn has passed its condition code
 668          * check, we take the IMPDEF choice to always report CV=1 COND=0xe
 669          * (which is also the required value for AArch64 traps).
 670          */
 671         syndrome = syn_fp_access_trap(1, 0xe, false);
 672         break;
 673     case CP_ACCESS_TRAP_FP_EL3:
 674         target_el = 3;
 675         syndrome = syn_fp_access_trap(1, 0xe, false);
 676         break;
 677     default:
 678         g_assert_not_reached();
 679     }
 680
 681     raise_exception(env, EXCP_UDEF, syndrome, target_el);
 682 }
 683
 684 void HELPER(set_cp_reg)(CPUARMState *env, void *rip, uint32_t value)
 685 {
 686     const ARMCPRegInfo *ri = rip;
 687
 688     ri->writefn(env, ri, value);
 689 }
 690
 691 uint32_t HELPER(get_cp_reg)(CPUARMState *env, void *rip)
 692 {
 693     const ARMCPRegInfo *ri = rip;
 694
 695     return ri->readfn(env, ri);
 696 }
 697
 698 void HELPER(set_cp_reg64)(CPUARMState *env, void *rip, uint64_t value)
 699 {
 700     const ARMCPRegInfo *ri = rip;
 701
 702     ri->writefn(env, ri, value);
 703 }
 704
 705 uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
 706 {
 707     const ARMCPRegInfo *ri = rip;
 708
 709     return ri->readfn(env, ri);
 710 }
 711
 712 void HELPER(msr_i_pstate)(CPUARMState *env, uint32_t op, uint32_t imm)
 713 {
 714     /* MSR_i to update PSTATE. This is OK from EL0 only if UMA is set.
 715      * Note that SPSel is never OK from EL0; we rely on handle_msr_i()
 716      * to catch that case at translate time.
 717      */
 718     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
 719         uint32_t syndrome = syn_aa64_sysregtrap(0, extract32(op, 0, 3),
 720                                                 extract32(op, 3, 3), 4,
 721                                                 imm, 0x1f, 0);
 722         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 723     }
 724
 725     switch (op) {
 726     case 0x05: /* SPSel */
 727         update_spsel(env, imm);
 728         break;
 729     case 0x1e: /* DAIFSet */
 730         env->daif |= (imm << 6) & PSTATE_DAIF;
 731         break;
 732     case 0x1f: /* DAIFClear */
 733         env->daif &= ~((imm << 6) & PSTATE_DAIF);
 734         break;
 735     default:
 736         g_assert_not_reached();
 737     }
 738 }
 739
 740 void HELPER(clear_pstate_ss)(CPUARMState *env)
 741 {
 742     env->pstate &= ~PSTATE_SS;
 743 }
 744
 745 void HELPER(pre_hvc)(CPUARMState *env)
 746 {
 747     ARMCPU *cpu = arm_env_get_cpu(env);
 748     int cur_el = arm_current_el(env);
 749     /* FIXME: Use actual secure state.  */
 750     bool secure = false;
 751     bool undef;
 752
 753     if (arm_is_psci_call(cpu, EXCP_HVC)) {
 754         /* If PSCI is enabled and this looks like a valid PSCI call then
 755          * that overrides the architecturally mandated HVC behaviour.
 756          */
 757         return;
 758     }
 759
 760     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 761         /* If EL2 doesn't exist, HVC always UNDEFs */
 762         undef = true;
 763     } else if (arm_feature(env, ARM_FEATURE_EL3)) {
 764         /* EL3.HCE has priority over EL2.HCD. */
 765         undef = !(env->cp15.scr_el3 & SCR_HCE);
 766     } else {
 767         undef = env->cp15.hcr_el2 & HCR_HCD;
 768     }
 769
 770     /* In ARMv7 and ARMv8/AArch32, HVC is undef in secure state.
 771      * For ARMv8/AArch64, HVC is allowed in EL3.
 772      * Note that we've already trapped HVC from EL0 at translation
 773      * time.
 774      */
 775     if (secure && (!is_a64(env) || cur_el == 1)) {
 776         undef = true;
 777     }
 778
 779     if (undef) {
 780         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 781                         exception_target_el(env));
 782     }
 783 }
 784
 785 void HELPER(pre_smc)(CPUARMState *env, uint32_t syndrome)
 786 {
 787     ARMCPU *cpu = arm_env_get_cpu(env);
 788     int cur_el = arm_current_el(env);
 789     bool secure = arm_is_secure(env);
 790     bool smd = env->cp15.scr_el3 & SCR_SMD;
 791     /* On ARMv8 with EL3 AArch64, SMD applies to both S and NS state.
 792      * On ARMv8 with EL3 AArch32, or ARMv7 with the Virtualization
 793      *  extensions, SMD only applies to NS state.
 794      * On ARMv7 without the Virtualization extensions, the SMD bit
 795      * doesn't exist, but we forbid the guest to set it to 1 in scr_write(),
 796      * so we need not special case this here.
 797      */
 798     bool undef = arm_feature(env, ARM_FEATURE_AARCH64) ? smd : smd && !secure;
 799
 800     if (arm_is_psci_call(cpu, EXCP_SMC)) {
 801         /* If PSCI is enabled and this looks like a valid PSCI call then
 802          * that overrides the architecturally mandated SMC behaviour.
 803          */
 804         return;
 805     }
 806
 807     if (!arm_feature(env, ARM_FEATURE_EL3)) {
 808         /* If we have no EL3 then SMC always UNDEFs */
 809         undef = true;
 810     } else if (!secure && cur_el == 1 && (env->cp15.hcr_el2 & HCR_TSC)) {
 811         /* In NS EL1, HCR controlled routing to EL2 has priority over SMD. */
 812         raise_exception(env, EXCP_HYP_TRAP, syndrome, 2);
 813     }
 814
 815     if (undef) {
 816         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 817                         exception_target_el(env));
 818     }
 819 }
 820
 821 static int el_from_spsr(uint32_t spsr)
 822 {
 823     /* Return the exception level that this SPSR is requesting a return to,
 824      * or -1 if it is invalid (an illegal return)
 825      */
 826     if (spsr & PSTATE_nRW) {
 827         switch (spsr & CPSR_M) {
 828         case ARM_CPU_MODE_USR:
 829             return 0;
 830         case ARM_CPU_MODE_HYP:
 831             return 2;
 832         case ARM_CPU_MODE_FIQ:
 833         case ARM_CPU_MODE_IRQ:
 834         case ARM_CPU_MODE_SVC:
 835         case ARM_CPU_MODE_ABT:
 836         case ARM_CPU_MODE_UND:
 837         case ARM_CPU_MODE_SYS:
 838             return 1;
 839         case ARM_CPU_MODE_MON:
 840             /* Returning to Mon from AArch64 is never possible,
 841              * so this is an illegal return.
 842              */
 843         default:
 844             return -1;
 845         }
 846     } else {
 847         if (extract32(spsr, 1, 1)) {
 848             /* Return with reserved M[1] bit set */
 849             return -1;
 850         }
 851         if (extract32(spsr, 0, 4) == 1) {
 852             /* return to EL0 with M[0] bit set */
 853             return -1;
 854         }
 855         return extract32(spsr, 2, 2);
 856     }
 857 }
 858
 859 void HELPER(exception_return)(CPUARMState *env)
 860 {
 861     int cur_el = arm_current_el(env);
 862     unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
 863     uint32_t spsr = env->banked_spsr[spsr_idx];
 864     int new_el;
 865     bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
 866
 867     aarch64_save_sp(env, cur_el);
 868
 869     env->exclusive_addr = -1;
 870
 871     /* We must squash the PSTATE.SS bit to zero unless both of the
 872      * following hold:
 873      *  1. debug exceptions are currently disabled
 874      *  2. singlestep will be active in the EL we return to
 875      * We check 1 here and 2 after we've done the pstate/cpsr write() to
 876      * transition to the EL we're going to.
 877      */
 878     if (arm_generate_debug_exceptions(env)) {
 879         spsr &= ~PSTATE_SS;
 880     }
 881
 882     new_el = el_from_spsr(spsr);
 883     if (new_el == -1) {
 884         goto illegal_return;
 885     }
 886     if (new_el > cur_el
 887         || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
 888         /* Disallow return to an EL which is unimplemented or higher
 889          * than the current one.
 890          */
 891         goto illegal_return;
 892     }
 893
 894     if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
 895         /* Return to an EL which is configured for a different register width */
 896         goto illegal_return;
 897     }
 898
 899     if (new_el == 2 && arm_is_secure_below_el3(env)) {
 900         /* Return to the non-existent secure-EL2 */
 901         goto illegal_return;
 902     }
 903
 904     if (new_el == 1 && (env->cp15.hcr_el2 & HCR_TGE)
 905         && !arm_is_secure_below_el3(env)) {
 906         goto illegal_return;
 907     }
 908
 909     if (!return_to_aa64) {
 910         env->aarch64 = 0;
 911         /* We do a raw CPSR write because aarch64_sync_64_to_32()
 912          * will sort the register banks out for us, and we've already
 913          * caught all the bad-mode cases in el_from_spsr().
 914          */
 915         cpsr_write(env, spsr, ~0, CPSRWriteRaw);
 916         if (!arm_singlestep_active(env)) {
 917             env->uncached_cpsr &= ~PSTATE_SS;
 918         }
 919         aarch64_sync_64_to_32(env);
 920
 921         if (spsr & CPSR_T) {
 922             env->regs[15] = env->elr_el[cur_el] & ~0x1;
 923         } else {
 924             env->regs[15] = env->elr_el[cur_el] & ~0x3;
 925         }
 926     } else {
 927         env->aarch64 = 1;
 928         pstate_write(env, spsr);
 929         if (!arm_singlestep_active(env)) {
 930             env->pstate &= ~PSTATE_SS;
 931         }
 932         aarch64_restore_sp(env, new_el);
 933         env->pc = env->elr_el[cur_el];
 934     }
 935
 936     return;
 937
 938 illegal_return:
 939     /* Illegal return events of various kinds have architecturally
 940      * mandated behaviour:
 941      * restore NZCV and DAIF from SPSR_ELx
 942      * set PSTATE.IL
 943      * restore PC from ELR_ELx
 944      * no change to exception level, execution state or stack pointer
 945      */
 946     env->pstate |= PSTATE_IL;
 947     env->pc = env->elr_el[cur_el];
 948     spsr &= PSTATE_NZCV | PSTATE_DAIF;
 949     spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
 950     pstate_write(env, spsr);
 951     if (!arm_singlestep_active(env)) {
 952         env->pstate &= ~PSTATE_SS;
 953     }
 954 }
 955
 956 /* Return true if the linked breakpoint entry lbn passes its checks */
 957 static bool linked_bp_matches(ARMCPU *cpu, int lbn)
 958 {
 959     CPUARMState *env = &cpu->env;
 960     uint64_t bcr = env->cp15.dbgbcr[lbn];
 961     int brps = extract32(cpu->dbgdidr, 24, 4);
 962     int ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
 963     int bt;
 964     uint32_t contextidr;
 965
 966     /* Links to unimplemented or non-context aware breakpoints are
 967      * CONSTRAINED UNPREDICTABLE: either behave as if disabled, or
 968      * as if linked to an UNKNOWN context-aware breakpoint (in which
 969      * case DBGWCR<n>_EL1.LBN must indicate that breakpoint).
 970      * We choose the former.
 971      */
 972     if (lbn > brps || lbn < (brps - ctx_cmps)) {
 973         return false;
 974     }
 975
 976     bcr = env->cp15.dbgbcr[lbn];
 977
 978     if (extract64(bcr, 0, 1) == 0) {
 979         /* Linked breakpoint disabled : generate no events */
 980         return false;
 981     }
 982
 983     bt = extract64(bcr, 20, 4);
 984
 985     /* We match the whole register even if this is AArch32 using the
 986      * short descriptor format (in which case it holds both PROCID and ASID),
 987      * since we don't implement the optional v7 context ID masking.
 988      */
 989     contextidr = extract64(env->cp15.contextidr_el[1], 0, 32);
 990
 991     switch (bt) {
 992     case 3: /* linked context ID match */
 993         if (arm_current_el(env) > 1) {
 994             /* Context matches never fire in EL2 or (AArch64) EL3 */
 995             return false;
 996         }
 997         return (contextidr == extract64(env->cp15.dbgbvr[lbn], 0, 32));
 998     case 5: /* linked address mismatch (reserved in AArch64) */
 999     case 9: /* linked VMID match (reserved if no EL2) */
1000     case 11: /* linked context ID and VMID match (reserved if no EL2) */
1001     default:
1002         /* Links to Unlinked context breakpoints must generate no
1003          * events; we choose to do the same for reserved values too.
1004          */
1005         return false;
1006     }
1007
1008     return false;
1009 }
1010
1011 static bool bp_wp_matches(ARMCPU *cpu, int n, bool is_wp)
1012 {
1013     CPUARMState *env = &cpu->env;
1014     uint64_t cr;
1015     int pac, hmc, ssc, wt, lbn;
1016     /* Note that for watchpoints the check is against the CPU security
1017      * state, not the S/NS attribute on the offending data access.
1018      */
1019     bool is_secure = arm_is_secure(env);
1020     int access_el = arm_current_el(env);
1021
1022     if (is_wp) {
1023         CPUWatchpoint *wp = env->cpu_watchpoint[n];
1024
1025         if (!wp || !(wp->flags & BP_WATCHPOINT_HIT)) {
1026             return false;
1027         }
1028         cr = env->cp15.dbgwcr[n];
1029         if (wp->hitattrs.user) {
1030             /* The LDRT/STRT/LDT/STT "unprivileged access" instructions should
1031              * match watchpoints as if they were accesses done at EL0, even if
1032              * the CPU is at EL1 or higher.
1033              */
1034             access_el = 0;
1035         }
1036     } else {
1037         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1038
1039         if (!env->cpu_breakpoint[n] || env->cpu_breakpoint[n]->pc != pc) {
1040             return false;
1041         }
1042         cr = env->cp15.dbgbcr[n];
1043     }
1044     /* The WATCHPOINT_HIT flag guarantees us that the watchpoint is
1045      * enabled and that the address and access type match; for breakpoints
1046      * we know the address matched; check the remaining fields, including
1047      * linked breakpoints. We rely on WCR and BCR having the same layout
1048      * for the LBN, SSC, HMC, PAC/PMC and is-linked fields.
1049      * Note that some combinations of {PAC, HMC, SSC} are reserved and
1050      * must act either like some valid combination or as if the watchpoint
1051      * were disabled. We choose the former, and use this together with
1052      * the fact that EL3 must always be Secure and EL2 must always be
1053      * Non-Secure to simplify the code slightly compared to the full
1054      * table in the ARM ARM.
1055      */
1056     pac = extract64(cr, 1, 2);
1057     hmc = extract64(cr, 13, 1);
1058     ssc = extract64(cr, 14, 2);
1059
1060     switch (ssc) {
1061     case 0:
1062         break;
1063     case 1:
1064     case 3:
1065         if (is_secure) {
1066             return false;
1067         }
1068         break;
1069     case 2:
1070         if (!is_secure) {
1071             return false;
1072         }
1073         break;
1074     }
1075
1076     switch (access_el) {
1077     case 3:
1078     case 2:
1079         if (!hmc) {
1080             return false;
1081         }
1082         break;
1083     case 1:
1084         if (extract32(pac, 0, 1) == 0) {
1085             return false;
1086         }
1087         break;
1088     case 0:
1089         if (extract32(pac, 1, 1) == 0) {
1090             return false;
1091         }
1092         break;
1093     default:
1094         g_assert_not_reached();
1095     }
1096
1097     wt = extract64(cr, 20, 1);
1098     lbn = extract64(cr, 16, 4);
1099
1100     if (wt && !linked_bp_matches(cpu, lbn)) {
1101         return false;
1102     }
1103
1104     return true;
1105 }
1106
1107 static bool check_watchpoints(ARMCPU *cpu)
1108 {
1109     CPUARMState *env = &cpu->env;
1110     int n;
1111
1112     /* If watchpoints are disabled globally or we can't take debug
1113      * exceptions here then watchpoint firings are ignored.
1114      */
1115     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1116         || !arm_generate_debug_exceptions(env)) {
1117         return false;
1118     }
1119
1120     for (n = 0; n < ARRAY_SIZE(env->cpu_watchpoint); n++) {
1121         if (bp_wp_matches(cpu, n, true)) {
1122             return true;
1123         }
1124     }
1125     return false;
1126 }
1127
1128 static bool check_breakpoints(ARMCPU *cpu)
1129 {
1130     CPUARMState *env = &cpu->env;
1131     int n;
1132
1133     /* If breakpoints are disabled globally or we can't take debug
1134      * exceptions here then breakpoint firings are ignored.
1135      */
1136     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1137         || !arm_generate_debug_exceptions(env)) {
1138         return false;
1139     }
1140
1141     for (n = 0; n < ARRAY_SIZE(env->cpu_breakpoint); n++) {
1142         if (bp_wp_matches(cpu, n, false)) {
1143             return true;
1144         }
1145     }
1146     return false;
1147 }
1148
1149 void HELPER(check_breakpoints)(CPUARMState *env)
1150 {
1151     ARMCPU *cpu = arm_env_get_cpu(env);
1152
1153     if (check_breakpoints(cpu)) {
1154         HELPER(exception_internal(env, EXCP_DEBUG));
1155     }
1156 }
1157
1158 bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
1159 {
1160     /* Called by core code when a CPU watchpoint fires; need to check if this
1161      * is also an architectural watchpoint match.
1162      */
1163     ARMCPU *cpu = ARM_CPU(cs);
1164
1165     return check_watchpoints(cpu);
1166 }
1167
1168 void arm_debug_excp_handler(CPUState *cs)
1169 {
1170     /* Called by core code when a watchpoint or breakpoint fires;
1171      * need to check which one and raise the appropriate exception.
1172      */
1173     ARMCPU *cpu = ARM_CPU(cs);
1174     CPUARMState *env = &cpu->env;
1175     CPUWatchpoint *wp_hit = cs->watchpoint_hit;
1176
1177     if (wp_hit) {
1178         if (wp_hit->flags & BP_CPU) {
1179             bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
1180             bool same_el = arm_debug_target_el(env) == arm_current_el(env);
1181
1182             cs->watchpoint_hit = NULL;
1183
1184             if (extended_addresses_enabled(env)) {
1185                 env->exception.fsr = (1 << 9) | 0x22;
1186             } else {
1187                 env->exception.fsr = 0x2;
1188             }
1189             env->exception.vaddress = wp_hit->hitaddr;
1190             raise_exception(env, EXCP_DATA_ABORT,
1191                     syn_watchpoint(same_el, 0, wnr),
1192                     arm_debug_target_el(env));
1193         }
1194     } else {
1195         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1196         bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
1197
1198         /* (1) GDB breakpoints should be handled first.
1199          * (2) Do not raise a CPU exception if no CPU breakpoint has fired,
1200          * since singlestep is also done by generating a debug internal
1201          * exception.
1202          */
1203         if (cpu_breakpoint_test(cs, pc, BP_GDB)
1204             || !cpu_breakpoint_test(cs, pc, BP_CPU)) {
1205             return;
1206         }
1207
1208         if (extended_addresses_enabled(env)) {
1209             env->exception.fsr = (1 << 9) | 0x22;
1210         } else {
1211             env->exception.fsr = 0x2;
1212         }
1213         /* FAR is UNKNOWN, so doesn't need setting */
1214         raise_exception(env, EXCP_PREFETCH_ABORT,
1215                         syn_breakpoint(same_el),
1216                         arm_debug_target_el(env));
1217     }
1218 }
1219
1220 /* ??? Flag setting arithmetic is awkward because we need to do comparisons.
1221    The only way to do that in TCG is a conditional branch, which clobbers
1222    all our temporaries.  For now implement these as helper functions.  */
1223
1224 /* Similarly for variable shift instructions.  */
1225
1226 uint32_t HELPER(shl_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1227 {
1228     int shift = i & 0xff;
1229     if (shift >= 32) {
1230         if (shift == 32)
1231             env->CF = x & 1;
1232         else
1233             env->CF = 0;
1234         return 0;
1235     } else if (shift != 0) {
1236         env->CF = (x >> (32 - shift)) & 1;
1237         return x << shift;
1238     }
1239     return x;
1240 }
1241
1242 uint32_t HELPER(shr_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1243 {
1244     int shift = i & 0xff;
1245     if (shift >= 32) {
1246         if (shift == 32)
1247             env->CF = (x >> 31) & 1;
1248         else
1249             env->CF = 0;
1250         return 0;
1251     } else if (shift != 0) {
1252         env->CF = (x >> (shift - 1)) & 1;
1253         return x >> shift;
1254     }
1255     return x;
1256 }
1257
1258 uint32_t HELPER(sar_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1259 {
1260     int shift = i & 0xff;
1261     if (shift >= 32) {
1262         env->CF = (x >> 31) & 1;
1263         return (int32_t)x >> 31;
1264     } else if (shift != 0) {
1265         env->CF = (x >> (shift - 1)) & 1;
1266         return (int32_t)x >> shift;
1267     }
1268     return x;
1269 }
1270
1271 uint32_t HELPER(ror_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1272 {
1273     int shift1, shift;
1274     shift1 = i & 0xff;
1275     shift = shift1 & 0x1f;
1276     if (shift == 0) {
1277         if (shift1 != 0)
1278             env->CF = (x >> 31) & 1;
1279         return x;
1280     } else {
1281         env->CF = (x >> (shift - 1)) & 1;
1282         return ((uint32_t)x >> shift) | (x << (32 - shift));
1283     }
1284 }