target-arm/op_helper.c

   1 /*
   2  *  ARM helper routines
   3  *
   4  *  Copyright (c) 2005-2007 CodeSourcery, LLC
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include "qemu/osdep.h"
  20 #include "cpu.h"
  21 #include "exec/helper-proto.h"
  22 #include "internals.h"
  23 #include "exec/exec-all.h"
  24 #include "exec/cpu_ldst.h"
  25
  26 #define SIGNBIT (uint32_t)0x80000000
  27 #define SIGNBIT64 ((uint64_t)1 << 63)
  28
  29 static void QEMU_NORETURN
  30 raise_exception(CPUARMState *env, uint32_t excp,
  31                 uint32_t syndrome, uint32_t target_el)
  32 {
  33     CPUState *cs = CPU(arm_env_get_cpu(env));
  34
  35     assert(!excp_is_internal(excp));
  36     cs->exception_index = excp;
  37     env->exception.syndrome = syndrome;
  38     env->exception.target_el = target_el;
  39     cpu_loop_exit(cs);
  40 }
  41
  42 static int exception_target_el(CPUARMState *env)
  43 {
  44     int target_el = MAX(1, arm_current_el(env));
  45
  46     /* No such thing as secure EL1 if EL3 is aarch32, so update the target EL
  47      * to EL3 in this case.
  48      */
  49     if (arm_is_secure(env) && !arm_el_is_aa64(env, 3) && target_el == 1) {
  50         target_el = 3;
  51     }
  52
  53     return target_el;
  54 }
  55
  56 uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
  57                           uint32_t rn, uint32_t maxindex)
  58 {
  59     uint32_t val;
  60     uint32_t tmp;
  61     int index;
  62     int shift;
  63     uint64_t *table;
  64     table = (uint64_t *)&env->vfp.regs[rn];
  65     val = 0;
  66     for (shift = 0; shift < 32; shift += 8) {
  67         index = (ireg >> shift) & 0xff;
  68         if (index < maxindex) {
  69             tmp = (table[index >> 3] >> ((index & 7) << 3)) & 0xff;
  70             val |= tmp << shift;
  71         } else {
  72             val |= def & (0xff << shift);
  73         }
  74     }
  75     return val;
  76 }
  77
  78 #if !defined(CONFIG_USER_ONLY)
  79
  80 static inline uint32_t merge_syn_data_abort(uint32_t template_syn,
  81                                             unsigned int target_el,
  82                                             bool same_el,
  83                                             bool s1ptw, bool is_write,
  84                                             int fsc)
  85 {
  86     uint32_t syn;
  87
  88     /* ISV is only set for data aborts routed to EL2 and
  89      * never for stage-1 page table walks faulting on stage 2.
  90      *
  91      * Furthermore, ISV is only set for certain kinds of load/stores.
  92      * If the template syndrome does not have ISV set, we should leave
  93      * it cleared.
  94      *
  95      * See ARMv8 specs, D7-1974:
  96      * ISS encoding for an exception from a Data Abort, the
  97      * ISV field.
  98      */
  99     if (!(template_syn & ARM_EL_ISV) || target_el != 2 || s1ptw) {
 100         syn = syn_data_abort_no_iss(same_el,
 101                                     0, 0, s1ptw, is_write, fsc);
 102     } else {
 103         /* Fields: IL, ISV, SAS, SSE, SRT, SF and AR come from the template
 104          * syndrome created at translation time.
 105          * Now we create the runtime syndrome with the remaining fields.
 106          */
 107         syn = syn_data_abort_with_iss(same_el,
 108                                       0, 0, 0, 0, 0,
 109                                       0, 0, s1ptw, is_write, fsc,
 110                                       false);
 111         /* Merge the runtime syndrome with the template syndrome.  */
 112         syn |= template_syn;
 113     }
 114     return syn;
 115 }
 116
 117 /* try to fill the TLB and return an exception if error. If retaddr is
 118  * NULL, it means that the function was called in C code (i.e. not
 119  * from generated code or from helper.c)
 120  */
 121 void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type,
 122               int mmu_idx, uintptr_t retaddr)
 123 {
 124     bool ret;
 125     uint32_t fsr = 0;
 126     ARMMMUFaultInfo fi = {};
 127
 128     ret = arm_tlb_fill(cs, addr, access_type, mmu_idx, &fsr, &fi);
 129     if (unlikely(ret)) {
 130         ARMCPU *cpu = ARM_CPU(cs);
 131         CPUARMState *env = &cpu->env;
 132         uint32_t syn, exc;
 133         unsigned int target_el;
 134         bool same_el;
 135
 136         if (retaddr) {
 137             /* now we have a real cpu fault */
 138             cpu_restore_state(cs, retaddr);
 139         }
 140
 141         target_el = exception_target_el(env);
 142         if (fi.stage2) {
 143             target_el = 2;
 144             env->cp15.hpfar_el2 = extract64(fi.s2addr, 12, 47) << 4;
 145         }
 146         same_el = arm_current_el(env) == target_el;
 147         /* AArch64 syndrome does not have an LPAE bit */
 148         syn = fsr & ~(1 << 9);
 149
 150         /* For insn and data aborts we assume there is no instruction syndrome
 151          * information; this is always true for exceptions reported to EL1.
 152          */
 153         if (access_type == MMU_INST_FETCH) {
 154             syn = syn_insn_abort(same_el, 0, fi.s1ptw, syn);
 155             exc = EXCP_PREFETCH_ABORT;
 156         } else {
 157             syn = merge_syn_data_abort(env->exception.syndrome, target_el,
 158                                        same_el, fi.s1ptw,
 159                                        access_type == MMU_DATA_STORE, syn);
 160             if (access_type == MMU_DATA_STORE
 161                 && arm_feature(env, ARM_FEATURE_V6)) {
 162                 fsr |= (1 << 11);
 163             }
 164             exc = EXCP_DATA_ABORT;
 165         }
 166
 167         env->exception.vaddress = addr;
 168         env->exception.fsr = fsr;
 169         raise_exception(env, exc, syn, target_el);
 170     }
 171 }
 172
 173 /* Raise a data fault alignment exception for the specified virtual address */
 174 void arm_cpu_do_unaligned_access(CPUState *cs, vaddr vaddr,
 175                                  MMUAccessType access_type,
 176                                  int mmu_idx, uintptr_t retaddr)
 177 {
 178     ARMCPU *cpu = ARM_CPU(cs);
 179     CPUARMState *env = &cpu->env;
 180     int target_el;
 181     bool same_el;
 182     uint32_t syn;
 183
 184     if (retaddr) {
 185         /* now we have a real cpu fault */
 186         cpu_restore_state(cs, retaddr);
 187     }
 188
 189     target_el = exception_target_el(env);
 190     same_el = (arm_current_el(env) == target_el);
 191
 192     env->exception.vaddress = vaddr;
 193
 194     /* the DFSR for an alignment fault depends on whether we're using
 195      * the LPAE long descriptor format, or the short descriptor format
 196      */
 197     if (arm_s1_regime_using_lpae_format(env, cpu_mmu_index(env, false))) {
 198         env->exception.fsr = 0x21;
 199     } else {
 200         env->exception.fsr = 0x1;
 201     }
 202
 203     if (access_type == MMU_DATA_STORE && arm_feature(env, ARM_FEATURE_V6)) {
 204         env->exception.fsr |= (1 << 11);
 205     }
 206
 207     syn = merge_syn_data_abort(env->exception.syndrome, target_el,
 208                                same_el, 0, access_type == MMU_DATA_STORE,
 209                                0x21);
 210     raise_exception(env, EXCP_DATA_ABORT, syn, target_el);
 211 }
 212
 213 #endif /* !defined(CONFIG_USER_ONLY) */
 214
 215 uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 216 {
 217     uint32_t res = a + b;
 218     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT))
 219         env->QF = 1;
 220     return res;
 221 }
 222
 223 uint32_t HELPER(add_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 224 {
 225     uint32_t res = a + b;
 226     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
 227         env->QF = 1;
 228         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 229     }
 230     return res;
 231 }
 232
 233 uint32_t HELPER(sub_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 234 {
 235     uint32_t res = a - b;
 236     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
 237         env->QF = 1;
 238         res = ~(((int32_t)a >> 31) ^ SIGNBIT);
 239     }
 240     return res;
 241 }
 242
 243 uint32_t HELPER(double_saturate)(CPUARMState *env, int32_t val)
 244 {
 245     uint32_t res;
 246     if (val >= 0x40000000) {
 247         res = ~SIGNBIT;
 248         env->QF = 1;
 249     } else if (val <= (int32_t)0xc0000000) {
 250         res = SIGNBIT;
 251         env->QF = 1;
 252     } else {
 253         res = val << 1;
 254     }
 255     return res;
 256 }
 257
 258 uint32_t HELPER(add_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 259 {
 260     uint32_t res = a + b;
 261     if (res < a) {
 262         env->QF = 1;
 263         res = ~0;
 264     }
 265     return res;
 266 }
 267
 268 uint32_t HELPER(sub_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 269 {
 270     uint32_t res = a - b;
 271     if (res > a) {
 272         env->QF = 1;
 273         res = 0;
 274     }
 275     return res;
 276 }
 277
 278 /* Signed saturation.  */
 279 static inline uint32_t do_ssat(CPUARMState *env, int32_t val, int shift)
 280 {
 281     int32_t top;
 282     uint32_t mask;
 283
 284     top = val >> shift;
 285     mask = (1u << shift) - 1;
 286     if (top > 0) {
 287         env->QF = 1;
 288         return mask;
 289     } else if (top < -1) {
 290         env->QF = 1;
 291         return ~mask;
 292     }
 293     return val;
 294 }
 295
 296 /* Unsigned saturation.  */
 297 static inline uint32_t do_usat(CPUARMState *env, int32_t val, int shift)
 298 {
 299     uint32_t max;
 300
 301     max = (1u << shift) - 1;
 302     if (val < 0) {
 303         env->QF = 1;
 304         return 0;
 305     } else if (val > max) {
 306         env->QF = 1;
 307         return max;
 308     }
 309     return val;
 310 }
 311
 312 /* Signed saturate.  */
 313 uint32_t HELPER(ssat)(CPUARMState *env, uint32_t x, uint32_t shift)
 314 {
 315     return do_ssat(env, x, shift);
 316 }
 317
 318 /* Dual halfword signed saturate.  */
 319 uint32_t HELPER(ssat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 320 {
 321     uint32_t res;
 322
 323     res = (uint16_t)do_ssat(env, (int16_t)x, shift);
 324     res |= do_ssat(env, ((int32_t)x) >> 16, shift) << 16;
 325     return res;
 326 }
 327
 328 /* Unsigned saturate.  */
 329 uint32_t HELPER(usat)(CPUARMState *env, uint32_t x, uint32_t shift)
 330 {
 331     return do_usat(env, x, shift);
 332 }
 333
 334 /* Dual halfword unsigned saturate.  */
 335 uint32_t HELPER(usat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 336 {
 337     uint32_t res;
 338
 339     res = (uint16_t)do_usat(env, (int16_t)x, shift);
 340     res |= do_usat(env, ((int32_t)x) >> 16, shift) << 16;
 341     return res;
 342 }
 343
 344 void HELPER(setend)(CPUARMState *env)
 345 {
 346     env->uncached_cpsr ^= CPSR_E;
 347 }
 348
 349 /* Function checks whether WFx (WFI/WFE) instructions are set up to be trapped.
 350  * The function returns the target EL (1-3) if the instruction is to be trapped;
 351  * otherwise it returns 0 indicating it is not trapped.
 352  */
 353 static inline int check_wfx_trap(CPUARMState *env, bool is_wfe)
 354 {
 355     int cur_el = arm_current_el(env);
 356     uint64_t mask;
 357
 358     /* If we are currently in EL0 then we need to check if SCTLR is set up for
 359      * WFx instructions being trapped to EL1. These trap bits don't exist in v7.
 360      */
 361     if (cur_el < 1 && arm_feature(env, ARM_FEATURE_V8)) {
 362         int target_el;
 363
 364         mask = is_wfe ? SCTLR_nTWE : SCTLR_nTWI;
 365         if (arm_is_secure_below_el3(env) && !arm_el_is_aa64(env, 3)) {
 366             /* Secure EL0 and Secure PL1 is at EL3 */
 367             target_el = 3;
 368         } else {
 369             target_el = 1;
 370         }
 371
 372         if (!(env->cp15.sctlr_el[target_el] & mask)) {
 373             return target_el;
 374         }
 375     }
 376
 377     /* We are not trapping to EL1; trap to EL2 if HCR_EL2 requires it
 378      * No need for ARM_FEATURE check as if HCR_EL2 doesn't exist the
 379      * bits will be zero indicating no trap.
 380      */
 381     if (cur_el < 2 && !arm_is_secure(env)) {
 382         mask = (is_wfe) ? HCR_TWE : HCR_TWI;
 383         if (env->cp15.hcr_el2 & mask) {
 384             return 2;
 385         }
 386     }
 387
 388     /* We are not trapping to EL1 or EL2; trap to EL3 if SCR_EL3 requires it */
 389     if (cur_el < 3) {
 390         mask = (is_wfe) ? SCR_TWE : SCR_TWI;
 391         if (env->cp15.scr_el3 & mask) {
 392             return 3;
 393         }
 394     }
 395
 396     return 0;
 397 }
 398
 399 void HELPER(wfi)(CPUARMState *env)
 400 {
 401     CPUState *cs = CPU(arm_env_get_cpu(env));
 402     int target_el = check_wfx_trap(env, false);
 403
 404     if (cpu_has_work(cs)) {
 405         /* Don't bother to go into our "low power state" if
 406          * we would just wake up immediately.
 407          */
 408         return;
 409     }
 410
 411     if (target_el) {
 412         env->pc -= 4;
 413         raise_exception(env, EXCP_UDEF, syn_wfx(1, 0xe, 0), target_el);
 414     }
 415
 416     cs->exception_index = EXCP_HLT;
 417     cs->halted = 1;
 418     cpu_loop_exit(cs);
 419 }
 420
 421 void QEMU_NORETURN HELPER(wfe)(CPUARMState *env)
 422 {
 423     /* This is a hint instruction that is semantically different
 424      * from YIELD even though we currently implement it identically.
 425      * Don't actually halt the CPU, just yield back to top
 426      * level loop. This is not going into a "low power state"
 427      * (ie halting until some event occurs), so we never take
 428      * a configurable trap to a different exception level.
 429      */
 430     HELPER(yield)(env);
 431 }
 432
 433 void QEMU_NORETURN HELPER(yield)(CPUARMState *env)
 434 {
 435     ARMCPU *cpu = arm_env_get_cpu(env);
 436     CPUState *cs = CPU(cpu);
 437
 438     /* This is a non-trappable hint instruction that generally indicates
 439      * that the guest is currently busy-looping. Yield control back to the
 440      * top level loop so that a more deserving VCPU has a chance to run.
 441      */
 442     cs->exception_index = EXCP_YIELD;
 443     cpu_loop_exit(cs);
 444 }
 445
 446 /* Raise an internal-to-QEMU exception. This is limited to only
 447  * those EXCP values which are special cases for QEMU to interrupt
 448  * execution and not to be used for exceptions which are passed to
 449  * the guest (those must all have syndrome information and thus should
 450  * use exception_with_syndrome).
 451  */
 452 void QEMU_NORETURN HELPER(exception_internal)(CPUARMState *env, uint32_t excp)
 453 {
 454     CPUState *cs = CPU(arm_env_get_cpu(env));
 455
 456     assert(excp_is_internal(excp));
 457     cs->exception_index = excp;
 458     cpu_loop_exit(cs);
 459 }
 460
 461 /* Raise an exception with the specified syndrome register value */
 462 void QEMU_NORETURN
 463 HELPER(exception_with_syndrome)(CPUARMState *env, uint32_t excp,
 464                                 uint32_t syndrome, uint32_t target_el)
 465 {
 466     raise_exception(env, excp, syndrome, target_el);
 467 }
 468
 469 uint32_t HELPER(cpsr_read)(CPUARMState *env)
 470 {
 471     return cpsr_read(env) & ~(CPSR_EXEC | CPSR_RESERVED);
 472 }
 473
 474 void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 475 {
 476     cpsr_write(env, val, mask, CPSRWriteByInstr);
 477 }
 478
 479 /* Write the CPSR for a 32-bit exception return */
 480 void HELPER(cpsr_write_eret)(CPUARMState *env, uint32_t val)
 481 {
 482     cpsr_write(env, val, CPSR_ERET_MASK, CPSRWriteExceptionReturn);
 483
 484     arm_call_el_change_hook(arm_env_get_cpu(env));
 485 }
 486
 487 /* Access to user mode registers from privileged modes.  */
 488 uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 489 {
 490     uint32_t val;
 491
 492     if (regno == 13) {
 493         val = env->banked_r13[BANK_USRSYS];
 494     } else if (regno == 14) {
 495         val = env->banked_r14[BANK_USRSYS];
 496     } else if (regno >= 8
 497                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 498         val = env->usr_regs[regno - 8];
 499     } else {
 500         val = env->regs[regno];
 501     }
 502     return val;
 503 }
 504
 505 void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 506 {
 507     if (regno == 13) {
 508         env->banked_r13[BANK_USRSYS] = val;
 509     } else if (regno == 14) {
 510         env->banked_r14[BANK_USRSYS] = val;
 511     } else if (regno >= 8
 512                && (env->uncached_cpsr & 0x1f) == ARM_CPU_MODE_FIQ) {
 513         env->usr_regs[regno - 8] = val;
 514     } else {
 515         env->regs[regno] = val;
 516     }
 517 }
 518
 519 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
 520 {
 521     if ((env->uncached_cpsr & CPSR_M) == mode) {
 522         env->regs[13] = val;
 523     } else {
 524         env->banked_r13[bank_number(mode)] = val;
 525     }
 526 }
 527
 528 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
 529 {
 530     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_SYS) {
 531         /* SRS instruction is UNPREDICTABLE from System mode; we UNDEF.
 532          * Other UNPREDICTABLE and UNDEF cases were caught at translate time.
 533          */
 534         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 535                         exception_target_el(env));
 536     }
 537
 538     if ((env->uncached_cpsr & CPSR_M) == mode) {
 539         return env->regs[13];
 540     } else {
 541         return env->banked_r13[bank_number(mode)];
 542     }
 543 }
 544
 545 static void msr_mrs_banked_exc_checks(CPUARMState *env, uint32_t tgtmode,
 546                                       uint32_t regno)
 547 {
 548     /* Raise an exception if the requested access is one of the UNPREDICTABLE
 549      * cases; otherwise return. This broadly corresponds to the pseudocode
 550      * BankedRegisterAccessValid() and SPSRAccessValid(),
 551      * except that we have already handled some cases at translate time.
 552      */
 553     int curmode = env->uncached_cpsr & CPSR_M;
 554
 555     if (curmode == tgtmode) {
 556         goto undef;
 557     }
 558
 559     if (tgtmode == ARM_CPU_MODE_USR) {
 560         switch (regno) {
 561         case 8 ... 12:
 562             if (curmode != ARM_CPU_MODE_FIQ) {
 563                 goto undef;
 564             }
 565             break;
 566         case 13:
 567             if (curmode == ARM_CPU_MODE_SYS) {
 568                 goto undef;
 569             }
 570             break;
 571         case 14:
 572             if (curmode == ARM_CPU_MODE_HYP || curmode == ARM_CPU_MODE_SYS) {
 573                 goto undef;
 574             }
 575             break;
 576         default:
 577             break;
 578         }
 579     }
 580
 581     if (tgtmode == ARM_CPU_MODE_HYP) {
 582         switch (regno) {
 583         case 17: /* ELR_Hyp */
 584             if (curmode != ARM_CPU_MODE_HYP && curmode != ARM_CPU_MODE_MON) {
 585                 goto undef;
 586             }
 587             break;
 588         default:
 589             if (curmode != ARM_CPU_MODE_MON) {
 590                 goto undef;
 591             }
 592             break;
 593         }
 594     }
 595
 596     return;
 597
 598 undef:
 599     raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 600                     exception_target_el(env));
 601 }
 602
 603 void HELPER(msr_banked)(CPUARMState *env, uint32_t value, uint32_t tgtmode,
 604                         uint32_t regno)
 605 {
 606     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 607
 608     switch (regno) {
 609     case 16: /* SPSRs */
 610         env->banked_spsr[bank_number(tgtmode)] = value;
 611         break;
 612     case 17: /* ELR_Hyp */
 613         env->elr_el[2] = value;
 614         break;
 615     case 13:
 616         env->banked_r13[bank_number(tgtmode)] = value;
 617         break;
 618     case 14:
 619         env->banked_r14[bank_number(tgtmode)] = value;
 620         break;
 621     case 8 ... 12:
 622         switch (tgtmode) {
 623         case ARM_CPU_MODE_USR:
 624             env->usr_regs[regno - 8] = value;
 625             break;
 626         case ARM_CPU_MODE_FIQ:
 627             env->fiq_regs[regno - 8] = value;
 628             break;
 629         default:
 630             g_assert_not_reached();
 631         }
 632         break;
 633     default:
 634         g_assert_not_reached();
 635     }
 636 }
 637
 638 uint32_t HELPER(mrs_banked)(CPUARMState *env, uint32_t tgtmode, uint32_t regno)
 639 {
 640     msr_mrs_banked_exc_checks(env, tgtmode, regno);
 641
 642     switch (regno) {
 643     case 16: /* SPSRs */
 644         return env->banked_spsr[bank_number(tgtmode)];
 645     case 17: /* ELR_Hyp */
 646         return env->elr_el[2];
 647     case 13:
 648         return env->banked_r13[bank_number(tgtmode)];
 649     case 14:
 650         return env->banked_r14[bank_number(tgtmode)];
 651     case 8 ... 12:
 652         switch (tgtmode) {
 653         case ARM_CPU_MODE_USR:
 654             return env->usr_regs[regno - 8];
 655         case ARM_CPU_MODE_FIQ:
 656             return env->fiq_regs[regno - 8];
 657         default:
 658             g_assert_not_reached();
 659         }
 660     default:
 661         g_assert_not_reached();
 662     }
 663 }
 664
 665 void HELPER(access_check_cp_reg)(CPUARMState *env, void *rip, uint32_t syndrome,
 666                                  uint32_t isread)
 667 {
 668     const ARMCPRegInfo *ri = rip;
 669     int target_el;
 670
 671     if (arm_feature(env, ARM_FEATURE_XSCALE) && ri->cp < 14
 672         && extract32(env->cp15.c15_cpar, ri->cp, 1) == 0) {
 673         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 674     }
 675
 676     if (!ri->accessfn) {
 677         return;
 678     }
 679
 680     switch (ri->accessfn(env, ri, isread)) {
 681     case CP_ACCESS_OK:
 682         return;
 683     case CP_ACCESS_TRAP:
 684         target_el = exception_target_el(env);
 685         break;
 686     case CP_ACCESS_TRAP_EL2:
 687         /* Requesting a trap to EL2 when we're in EL3 or S-EL0/1 is
 688          * a bug in the access function.
 689          */
 690         assert(!arm_is_secure(env) && arm_current_el(env) != 3);
 691         target_el = 2;
 692         break;
 693     case CP_ACCESS_TRAP_EL3:
 694         target_el = 3;
 695         break;
 696     case CP_ACCESS_TRAP_UNCATEGORIZED:
 697         target_el = exception_target_el(env);
 698         syndrome = syn_uncategorized();
 699         break;
 700     case CP_ACCESS_TRAP_UNCATEGORIZED_EL2:
 701         target_el = 2;
 702         syndrome = syn_uncategorized();
 703         break;
 704     case CP_ACCESS_TRAP_UNCATEGORIZED_EL3:
 705         target_el = 3;
 706         syndrome = syn_uncategorized();
 707         break;
 708     case CP_ACCESS_TRAP_FP_EL2:
 709         target_el = 2;
 710         /* Since we are an implementation that takes exceptions on a trapped
 711          * conditional insn only if the insn has passed its condition code
 712          * check, we take the IMPDEF choice to always report CV=1 COND=0xe
 713          * (which is also the required value for AArch64 traps).
 714          */
 715         syndrome = syn_fp_access_trap(1, 0xe, false);
 716         break;
 717     case CP_ACCESS_TRAP_FP_EL3:
 718         target_el = 3;
 719         syndrome = syn_fp_access_trap(1, 0xe, false);
 720         break;
 721     default:
 722         g_assert_not_reached();
 723     }
 724
 725     raise_exception(env, EXCP_UDEF, syndrome, target_el);
 726 }
 727
 728 void HELPER(set_cp_reg)(CPUARMState *env, void *rip, uint32_t value)
 729 {
 730     const ARMCPRegInfo *ri = rip;
 731
 732     ri->writefn(env, ri, value);
 733 }
 734
 735 uint32_t HELPER(get_cp_reg)(CPUARMState *env, void *rip)
 736 {
 737     const ARMCPRegInfo *ri = rip;
 738
 739     return ri->readfn(env, ri);
 740 }
 741
 742 void HELPER(set_cp_reg64)(CPUARMState *env, void *rip, uint64_t value)
 743 {
 744     const ARMCPRegInfo *ri = rip;
 745
 746     ri->writefn(env, ri, value);
 747 }
 748
 749 uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
 750 {
 751     const ARMCPRegInfo *ri = rip;
 752
 753     return ri->readfn(env, ri);
 754 }
 755
 756 void HELPER(msr_i_pstate)(CPUARMState *env, uint32_t op, uint32_t imm)
 757 {
 758     /* MSR_i to update PSTATE. This is OK from EL0 only if UMA is set.
 759      * Note that SPSel is never OK from EL0; we rely on handle_msr_i()
 760      * to catch that case at translate time.
 761      */
 762     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
 763         uint32_t syndrome = syn_aa64_sysregtrap(0, extract32(op, 0, 3),
 764                                                 extract32(op, 3, 3), 4,
 765                                                 imm, 0x1f, 0);
 766         raise_exception(env, EXCP_UDEF, syndrome, exception_target_el(env));
 767     }
 768
 769     switch (op) {
 770     case 0x05: /* SPSel */
 771         update_spsel(env, imm);
 772         break;
 773     case 0x1e: /* DAIFSet */
 774         env->daif |= (imm << 6) & PSTATE_DAIF;
 775         break;
 776     case 0x1f: /* DAIFClear */
 777         env->daif &= ~((imm << 6) & PSTATE_DAIF);
 778         break;
 779     default:
 780         g_assert_not_reached();
 781     }
 782 }
 783
 784 void HELPER(clear_pstate_ss)(CPUARMState *env)
 785 {
 786     env->pstate &= ~PSTATE_SS;
 787 }
 788
 789 void HELPER(pre_hvc)(CPUARMState *env)
 790 {
 791     ARMCPU *cpu = arm_env_get_cpu(env);
 792     int cur_el = arm_current_el(env);
 793     /* FIXME: Use actual secure state.  */
 794     bool secure = false;
 795     bool undef;
 796
 797     if (arm_is_psci_call(cpu, EXCP_HVC)) {
 798         /* If PSCI is enabled and this looks like a valid PSCI call then
 799          * that overrides the architecturally mandated HVC behaviour.
 800          */
 801         return;
 802     }
 803
 804     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 805         /* If EL2 doesn't exist, HVC always UNDEFs */
 806         undef = true;
 807     } else if (arm_feature(env, ARM_FEATURE_EL3)) {
 808         /* EL3.HCE has priority over EL2.HCD. */
 809         undef = !(env->cp15.scr_el3 & SCR_HCE);
 810     } else {
 811         undef = env->cp15.hcr_el2 & HCR_HCD;
 812     }
 813
 814     /* In ARMv7 and ARMv8/AArch32, HVC is undef in secure state.
 815      * For ARMv8/AArch64, HVC is allowed in EL3.
 816      * Note that we've already trapped HVC from EL0 at translation
 817      * time.
 818      */
 819     if (secure && (!is_a64(env) || cur_el == 1)) {
 820         undef = true;
 821     }
 822
 823     if (undef) {
 824         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 825                         exception_target_el(env));
 826     }
 827 }
 828
 829 void HELPER(pre_smc)(CPUARMState *env, uint32_t syndrome)
 830 {
 831     ARMCPU *cpu = arm_env_get_cpu(env);
 832     int cur_el = arm_current_el(env);
 833     bool secure = arm_is_secure(env);
 834     bool smd = env->cp15.scr_el3 & SCR_SMD;
 835     /* On ARMv8 with EL3 AArch64, SMD applies to both S and NS state.
 836      * On ARMv8 with EL3 AArch32, or ARMv7 with the Virtualization
 837      *  extensions, SMD only applies to NS state.
 838      * On ARMv7 without the Virtualization extensions, the SMD bit
 839      * doesn't exist, but we forbid the guest to set it to 1 in scr_write(),
 840      * so we need not special case this here.
 841      */
 842     bool undef = arm_feature(env, ARM_FEATURE_AARCH64) ? smd : smd && !secure;
 843
 844     if (arm_is_psci_call(cpu, EXCP_SMC)) {
 845         /* If PSCI is enabled and this looks like a valid PSCI call then
 846          * that overrides the architecturally mandated SMC behaviour.
 847          */
 848         return;
 849     }
 850
 851     if (!arm_feature(env, ARM_FEATURE_EL3)) {
 852         /* If we have no EL3 then SMC always UNDEFs */
 853         undef = true;
 854     } else if (!secure && cur_el == 1 && (env->cp15.hcr_el2 & HCR_TSC)) {
 855         /* In NS EL1, HCR controlled routing to EL2 has priority over SMD. */
 856         raise_exception(env, EXCP_HYP_TRAP, syndrome, 2);
 857     }
 858
 859     if (undef) {
 860         raise_exception(env, EXCP_UDEF, syn_uncategorized(),
 861                         exception_target_el(env));
 862     }
 863 }
 864
 865 static int el_from_spsr(uint32_t spsr)
 866 {
 867     /* Return the exception level that this SPSR is requesting a return to,
 868      * or -1 if it is invalid (an illegal return)
 869      */
 870     if (spsr & PSTATE_nRW) {
 871         switch (spsr & CPSR_M) {
 872         case ARM_CPU_MODE_USR:
 873             return 0;
 874         case ARM_CPU_MODE_HYP:
 875             return 2;
 876         case ARM_CPU_MODE_FIQ:
 877         case ARM_CPU_MODE_IRQ:
 878         case ARM_CPU_MODE_SVC:
 879         case ARM_CPU_MODE_ABT:
 880         case ARM_CPU_MODE_UND:
 881         case ARM_CPU_MODE_SYS:
 882             return 1;
 883         case ARM_CPU_MODE_MON:
 884             /* Returning to Mon from AArch64 is never possible,
 885              * so this is an illegal return.
 886              */
 887         default:
 888             return -1;
 889         }
 890     } else {
 891         if (extract32(spsr, 1, 1)) {
 892             /* Return with reserved M[1] bit set */
 893             return -1;
 894         }
 895         if (extract32(spsr, 0, 4) == 1) {
 896             /* return to EL0 with M[0] bit set */
 897             return -1;
 898         }
 899         return extract32(spsr, 2, 2);
 900     }
 901 }
 902
 903 void HELPER(exception_return)(CPUARMState *env)
 904 {
 905     int cur_el = arm_current_el(env);
 906     unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el);
 907     uint32_t spsr = env->banked_spsr[spsr_idx];
 908     int new_el;
 909     bool return_to_aa64 = (spsr & PSTATE_nRW) == 0;
 910
 911     aarch64_save_sp(env, cur_el);
 912
 913     env->exclusive_addr = -1;
 914
 915     /* We must squash the PSTATE.SS bit to zero unless both of the
 916      * following hold:
 917      *  1. debug exceptions are currently disabled
 918      *  2. singlestep will be active in the EL we return to
 919      * We check 1 here and 2 after we've done the pstate/cpsr write() to
 920      * transition to the EL we're going to.
 921      */
 922     if (arm_generate_debug_exceptions(env)) {
 923         spsr &= ~PSTATE_SS;
 924     }
 925
 926     new_el = el_from_spsr(spsr);
 927     if (new_el == -1) {
 928         goto illegal_return;
 929     }
 930     if (new_el > cur_el
 931         || (new_el == 2 && !arm_feature(env, ARM_FEATURE_EL2))) {
 932         /* Disallow return to an EL which is unimplemented or higher
 933          * than the current one.
 934          */
 935         goto illegal_return;
 936     }
 937
 938     if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) {
 939         /* Return to an EL which is configured for a different register width */
 940         goto illegal_return;
 941     }
 942
 943     if (new_el == 2 && arm_is_secure_below_el3(env)) {
 944         /* Return to the non-existent secure-EL2 */
 945         goto illegal_return;
 946     }
 947
 948     if (new_el == 1 && (env->cp15.hcr_el2 & HCR_TGE)
 949         && !arm_is_secure_below_el3(env)) {
 950         goto illegal_return;
 951     }
 952
 953     if (!return_to_aa64) {
 954         env->aarch64 = 0;
 955         /* We do a raw CPSR write because aarch64_sync_64_to_32()
 956          * will sort the register banks out for us, and we've already
 957          * caught all the bad-mode cases in el_from_spsr().
 958          */
 959         cpsr_write(env, spsr, ~0, CPSRWriteRaw);
 960         if (!arm_singlestep_active(env)) {
 961             env->uncached_cpsr &= ~PSTATE_SS;
 962         }
 963         aarch64_sync_64_to_32(env);
 964
 965         if (spsr & CPSR_T) {
 966             env->regs[15] = env->elr_el[cur_el] & ~0x1;
 967         } else {
 968             env->regs[15] = env->elr_el[cur_el] & ~0x3;
 969         }
 970     } else {
 971         env->aarch64 = 1;
 972         pstate_write(env, spsr);
 973         if (!arm_singlestep_active(env)) {
 974             env->pstate &= ~PSTATE_SS;
 975         }
 976         aarch64_restore_sp(env, new_el);
 977         env->pc = env->elr_el[cur_el];
 978     }
 979
 980     arm_call_el_change_hook(arm_env_get_cpu(env));
 981
 982     return;
 983
 984 illegal_return:
 985     /* Illegal return events of various kinds have architecturally
 986      * mandated behaviour:
 987      * restore NZCV and DAIF from SPSR_ELx
 988      * set PSTATE.IL
 989      * restore PC from ELR_ELx
 990      * no change to exception level, execution state or stack pointer
 991      */
 992     env->pstate |= PSTATE_IL;
 993     env->pc = env->elr_el[cur_el];
 994     spsr &= PSTATE_NZCV | PSTATE_DAIF;
 995     spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF);
 996     pstate_write(env, spsr);
 997     if (!arm_singlestep_active(env)) {
 998         env->pstate &= ~PSTATE_SS;
 999     }
1000 }
1001
1002 /* Return true if the linked breakpoint entry lbn passes its checks */
1003 static bool linked_bp_matches(ARMCPU *cpu, int lbn)
1004 {
1005     CPUARMState *env = &cpu->env;
1006     uint64_t bcr = env->cp15.dbgbcr[lbn];
1007     int brps = extract32(cpu->dbgdidr, 24, 4);
1008     int ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
1009     int bt;
1010     uint32_t contextidr;
1011
1012     /* Links to unimplemented or non-context aware breakpoints are
1013      * CONSTRAINED UNPREDICTABLE: either behave as if disabled, or
1014      * as if linked to an UNKNOWN context-aware breakpoint (in which
1015      * case DBGWCR<n>_EL1.LBN must indicate that breakpoint).
1016      * We choose the former.
1017      */
1018     if (lbn > brps || lbn < (brps - ctx_cmps)) {
1019         return false;
1020     }
1021
1022     bcr = env->cp15.dbgbcr[lbn];
1023
1024     if (extract64(bcr, 0, 1) == 0) {
1025         /* Linked breakpoint disabled : generate no events */
1026         return false;
1027     }
1028
1029     bt = extract64(bcr, 20, 4);
1030
1031     /* We match the whole register even if this is AArch32 using the
1032      * short descriptor format (in which case it holds both PROCID and ASID),
1033      * since we don't implement the optional v7 context ID masking.
1034      */
1035     contextidr = extract64(env->cp15.contextidr_el[1], 0, 32);
1036
1037     switch (bt) {
1038     case 3: /* linked context ID match */
1039         if (arm_current_el(env) > 1) {
1040             /* Context matches never fire in EL2 or (AArch64) EL3 */
1041             return false;
1042         }
1043         return (contextidr == extract64(env->cp15.dbgbvr[lbn], 0, 32));
1044     case 5: /* linked address mismatch (reserved in AArch64) */
1045     case 9: /* linked VMID match (reserved if no EL2) */
1046     case 11: /* linked context ID and VMID match (reserved if no EL2) */
1047     default:
1048         /* Links to Unlinked context breakpoints must generate no
1049          * events; we choose to do the same for reserved values too.
1050          */
1051         return false;
1052     }
1053
1054     return false;
1055 }
1056
1057 static bool bp_wp_matches(ARMCPU *cpu, int n, bool is_wp)
1058 {
1059     CPUARMState *env = &cpu->env;
1060     uint64_t cr;
1061     int pac, hmc, ssc, wt, lbn;
1062     /* Note that for watchpoints the check is against the CPU security
1063      * state, not the S/NS attribute on the offending data access.
1064      */
1065     bool is_secure = arm_is_secure(env);
1066     int access_el = arm_current_el(env);
1067
1068     if (is_wp) {
1069         CPUWatchpoint *wp = env->cpu_watchpoint[n];
1070
1071         if (!wp || !(wp->flags & BP_WATCHPOINT_HIT)) {
1072             return false;
1073         }
1074         cr = env->cp15.dbgwcr[n];
1075         if (wp->hitattrs.user) {
1076             /* The LDRT/STRT/LDT/STT "unprivileged access" instructions should
1077              * match watchpoints as if they were accesses done at EL0, even if
1078              * the CPU is at EL1 or higher.
1079              */
1080             access_el = 0;
1081         }
1082     } else {
1083         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1084
1085         if (!env->cpu_breakpoint[n] || env->cpu_breakpoint[n]->pc != pc) {
1086             return false;
1087         }
1088         cr = env->cp15.dbgbcr[n];
1089     }
1090     /* The WATCHPOINT_HIT flag guarantees us that the watchpoint is
1091      * enabled and that the address and access type match; for breakpoints
1092      * we know the address matched; check the remaining fields, including
1093      * linked breakpoints. We rely on WCR and BCR having the same layout
1094      * for the LBN, SSC, HMC, PAC/PMC and is-linked fields.
1095      * Note that some combinations of {PAC, HMC, SSC} are reserved and
1096      * must act either like some valid combination or as if the watchpoint
1097      * were disabled. We choose the former, and use this together with
1098      * the fact that EL3 must always be Secure and EL2 must always be
1099      * Non-Secure to simplify the code slightly compared to the full
1100      * table in the ARM ARM.
1101      */
1102     pac = extract64(cr, 1, 2);
1103     hmc = extract64(cr, 13, 1);
1104     ssc = extract64(cr, 14, 2);
1105
1106     switch (ssc) {
1107     case 0:
1108         break;
1109     case 1:
1110     case 3:
1111         if (is_secure) {
1112             return false;
1113         }
1114         break;
1115     case 2:
1116         if (!is_secure) {
1117             return false;
1118         }
1119         break;
1120     }
1121
1122     switch (access_el) {
1123     case 3:
1124     case 2:
1125         if (!hmc) {
1126             return false;
1127         }
1128         break;
1129     case 1:
1130         if (extract32(pac, 0, 1) == 0) {
1131             return false;
1132         }
1133         break;
1134     case 0:
1135         if (extract32(pac, 1, 1) == 0) {
1136             return false;
1137         }
1138         break;
1139     default:
1140         g_assert_not_reached();
1141     }
1142
1143     wt = extract64(cr, 20, 1);
1144     lbn = extract64(cr, 16, 4);
1145
1146     if (wt && !linked_bp_matches(cpu, lbn)) {
1147         return false;
1148     }
1149
1150     return true;
1151 }
1152
1153 static bool check_watchpoints(ARMCPU *cpu)
1154 {
1155     CPUARMState *env = &cpu->env;
1156     int n;
1157
1158     /* If watchpoints are disabled globally or we can't take debug
1159      * exceptions here then watchpoint firings are ignored.
1160      */
1161     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1162         || !arm_generate_debug_exceptions(env)) {
1163         return false;
1164     }
1165
1166     for (n = 0; n < ARRAY_SIZE(env->cpu_watchpoint); n++) {
1167         if (bp_wp_matches(cpu, n, true)) {
1168             return true;
1169         }
1170     }
1171     return false;
1172 }
1173
1174 static bool check_breakpoints(ARMCPU *cpu)
1175 {
1176     CPUARMState *env = &cpu->env;
1177     int n;
1178
1179     /* If breakpoints are disabled globally or we can't take debug
1180      * exceptions here then breakpoint firings are ignored.
1181      */
1182     if (extract32(env->cp15.mdscr_el1, 15, 1) == 0
1183         || !arm_generate_debug_exceptions(env)) {
1184         return false;
1185     }
1186
1187     for (n = 0; n < ARRAY_SIZE(env->cpu_breakpoint); n++) {
1188         if (bp_wp_matches(cpu, n, false)) {
1189             return true;
1190         }
1191     }
1192     return false;
1193 }
1194
1195 void HELPER(check_breakpoints)(CPUARMState *env)
1196 {
1197     ARMCPU *cpu = arm_env_get_cpu(env);
1198
1199     if (check_breakpoints(cpu)) {
1200         HELPER(exception_internal(env, EXCP_DEBUG));
1201     }
1202 }
1203
1204 bool arm_debug_check_watchpoint(CPUState *cs, CPUWatchpoint *wp)
1205 {
1206     /* Called by core code when a CPU watchpoint fires; need to check if this
1207      * is also an architectural watchpoint match.
1208      */
1209     ARMCPU *cpu = ARM_CPU(cs);
1210
1211     return check_watchpoints(cpu);
1212 }
1213
1214 void arm_debug_excp_handler(CPUState *cs)
1215 {
1216     /* Called by core code when a watchpoint or breakpoint fires;
1217      * need to check which one and raise the appropriate exception.
1218      */
1219     ARMCPU *cpu = ARM_CPU(cs);
1220     CPUARMState *env = &cpu->env;
1221     CPUWatchpoint *wp_hit = cs->watchpoint_hit;
1222
1223     if (wp_hit) {
1224         if (wp_hit->flags & BP_CPU) {
1225             bool wnr = (wp_hit->flags & BP_WATCHPOINT_HIT_WRITE) != 0;
1226             bool same_el = arm_debug_target_el(env) == arm_current_el(env);
1227
1228             cs->watchpoint_hit = NULL;
1229
1230             if (extended_addresses_enabled(env)) {
1231                 env->exception.fsr = (1 << 9) | 0x22;
1232             } else {
1233                 env->exception.fsr = 0x2;
1234             }
1235             env->exception.vaddress = wp_hit->hitaddr;
1236             raise_exception(env, EXCP_DATA_ABORT,
1237                     syn_watchpoint(same_el, 0, wnr),
1238                     arm_debug_target_el(env));
1239         }
1240     } else {
1241         uint64_t pc = is_a64(env) ? env->pc : env->regs[15];
1242         bool same_el = (arm_debug_target_el(env) == arm_current_el(env));
1243
1244         /* (1) GDB breakpoints should be handled first.
1245          * (2) Do not raise a CPU exception if no CPU breakpoint has fired,
1246          * since singlestep is also done by generating a debug internal
1247          * exception.
1248          */
1249         if (cpu_breakpoint_test(cs, pc, BP_GDB)
1250             || !cpu_breakpoint_test(cs, pc, BP_CPU)) {
1251             return;
1252         }
1253
1254         if (extended_addresses_enabled(env)) {
1255             env->exception.fsr = (1 << 9) | 0x22;
1256         } else {
1257             env->exception.fsr = 0x2;
1258         }
1259         /* FAR is UNKNOWN, so doesn't need setting */
1260         raise_exception(env, EXCP_PREFETCH_ABORT,
1261                         syn_breakpoint(same_el),
1262                         arm_debug_target_el(env));
1263     }
1264 }
1265
1266 /* ??? Flag setting arithmetic is awkward because we need to do comparisons.
1267    The only way to do that in TCG is a conditional branch, which clobbers
1268    all our temporaries.  For now implement these as helper functions.  */
1269
1270 /* Similarly for variable shift instructions.  */
1271
1272 uint32_t HELPER(shl_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1273 {
1274     int shift = i & 0xff;
1275     if (shift >= 32) {
1276         if (shift == 32)
1277             env->CF = x & 1;
1278         else
1279             env->CF = 0;
1280         return 0;
1281     } else if (shift != 0) {
1282         env->CF = (x >> (32 - shift)) & 1;
1283         return x << shift;
1284     }
1285     return x;
1286 }
1287
1288 uint32_t HELPER(shr_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1289 {
1290     int shift = i & 0xff;
1291     if (shift >= 32) {
1292         if (shift == 32)
1293             env->CF = (x >> 31) & 1;
1294         else
1295             env->CF = 0;
1296         return 0;
1297     } else if (shift != 0) {
1298         env->CF = (x >> (shift - 1)) & 1;
1299         return x >> shift;
1300     }
1301     return x;
1302 }
1303
1304 uint32_t HELPER(sar_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1305 {
1306     int shift = i & 0xff;
1307     if (shift >= 32) {
1308         env->CF = (x >> 31) & 1;
1309         return (int32_t)x >> 31;
1310     } else if (shift != 0) {
1311         env->CF = (x >> (shift - 1)) & 1;
1312         return (int32_t)x >> shift;
1313     }
1314     return x;
1315 }
1316
1317 uint32_t HELPER(ror_cc)(CPUARMState *env, uint32_t x, uint32_t i)
1318 {
1319     int shift1, shift;
1320     shift1 = i & 0xff;
1321     shift = shift1 & 0x1f;
1322     if (shift == 0) {
1323         if (shift1 != 0)
1324             env->CF = (x >> 31) & 1;
1325         return x;
1326     } else {
1327         env->CF = (x >> (shift - 1)) & 1;
1328         return ((uint32_t)x >> shift) | (x << (32 - shift));
1329     }
1330 }