target-arm/helper.c

   1 #include "qemu/osdep.h"
   2 #include "cpu.h"
   3 #include "internals.h"
   4 #include "exec/gdbstub.h"
   5 #include "exec/helper-proto.h"
   6 #include "qemu/host-utils.h"
   7 #include "sysemu/arch_init.h"
   8 #include "sysemu/sysemu.h"
   9 #include "qemu/bitops.h"
  10 #include "qemu/crc32c.h"
  11 #include "exec/cpu_ldst.h"
  12 #include "arm_ldst.h"
  13 #include <zlib.h> /* For crc32 */
  14 #include "exec/semihost.h"
  15 #include "sysemu/kvm.h"
  16
  17 #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */
  18
  19 #ifndef CONFIG_USER_ONLY
  20 static bool get_phys_addr(CPUARMState *env, target_ulong address,
  21                           int access_type, ARMMMUIdx mmu_idx,
  22                           hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
  23                           target_ulong *page_size, uint32_t *fsr,
  24                           ARMMMUFaultInfo *fi);
  25
  26 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
  27                                int access_type, ARMMMUIdx mmu_idx,
  28                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
  29                                target_ulong *page_size_ptr, uint32_t *fsr,
  30                                ARMMMUFaultInfo *fi);
  31
  32 /* Definitions for the PMCCNTR and PMCR registers */
  33 #define PMCRD   0x8
  34 #define PMCRC   0x4
  35 #define PMCRE   0x1
  36 #endif
  37
  38 static int vfp_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  39 {
  40     int nregs;
  41
  42     /* VFP data registers are always little-endian.  */
  43     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  44     if (reg < nregs) {
  45         stfq_le_p(buf, env->vfp.regs[reg]);
  46         return 8;
  47     }
  48     if (arm_feature(env, ARM_FEATURE_NEON)) {
  49         /* Aliases for Q regs.  */
  50         nregs += 16;
  51         if (reg < nregs) {
  52             stfq_le_p(buf, env->vfp.regs[(reg - 32) * 2]);
  53             stfq_le_p(buf + 8, env->vfp.regs[(reg - 32) * 2 + 1]);
  54             return 16;
  55         }
  56     }
  57     switch (reg - nregs) {
  58     case 0: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSID]); return 4;
  59     case 1: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSCR]); return 4;
  60     case 2: stl_p(buf, env->vfp.xregs[ARM_VFP_FPEXC]); return 4;
  61     }
  62     return 0;
  63 }
  64
  65 static int vfp_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
  66 {
  67     int nregs;
  68
  69     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  70     if (reg < nregs) {
  71         env->vfp.regs[reg] = ldfq_le_p(buf);
  72         return 8;
  73     }
  74     if (arm_feature(env, ARM_FEATURE_NEON)) {
  75         nregs += 16;
  76         if (reg < nregs) {
  77             env->vfp.regs[(reg - 32) * 2] = ldfq_le_p(buf);
  78             env->vfp.regs[(reg - 32) * 2 + 1] = ldfq_le_p(buf + 8);
  79             return 16;
  80         }
  81     }
  82     switch (reg - nregs) {
  83     case 0: env->vfp.xregs[ARM_VFP_FPSID] = ldl_p(buf); return 4;
  84     case 1: env->vfp.xregs[ARM_VFP_FPSCR] = ldl_p(buf); return 4;
  85     case 2: env->vfp.xregs[ARM_VFP_FPEXC] = ldl_p(buf) & (1 << 30); return 4;
  86     }
  87     return 0;
  88 }
  89
  90 static int aarch64_fpu_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  91 {
  92     switch (reg) {
  93     case 0 ... 31:
  94         /* 128 bit FP register */
  95         stfq_le_p(buf, env->vfp.regs[reg * 2]);
  96         stfq_le_p(buf + 8, env->vfp.regs[reg * 2 + 1]);
  97         return 16;
  98     case 32:
  99         /* FPSR */
 100         stl_p(buf, vfp_get_fpsr(env));
 101         return 4;
 102     case 33:
 103         /* FPCR */
 104         stl_p(buf, vfp_get_fpcr(env));
 105         return 4;
 106     default:
 107         return 0;
 108     }
 109 }
 110
 111 static int aarch64_fpu_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
 112 {
 113     switch (reg) {
 114     case 0 ... 31:
 115         /* 128 bit FP register */
 116         env->vfp.regs[reg * 2] = ldfq_le_p(buf);
 117         env->vfp.regs[reg * 2 + 1] = ldfq_le_p(buf + 8);
 118         return 16;
 119     case 32:
 120         /* FPSR */
 121         vfp_set_fpsr(env, ldl_p(buf));
 122         return 4;
 123     case 33:
 124         /* FPCR */
 125         vfp_set_fpcr(env, ldl_p(buf));
 126         return 4;
 127     default:
 128         return 0;
 129     }
 130 }
 131
 132 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
 133 {
 134     assert(ri->fieldoffset);
 135     if (cpreg_field_is_64bit(ri)) {
 136         return CPREG_FIELD64(env, ri);
 137     } else {
 138         return CPREG_FIELD32(env, ri);
 139     }
 140 }
 141
 142 static void raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
 143                       uint64_t value)
 144 {
 145     assert(ri->fieldoffset);
 146     if (cpreg_field_is_64bit(ri)) {
 147         CPREG_FIELD64(env, ri) = value;
 148     } else {
 149         CPREG_FIELD32(env, ri) = value;
 150     }
 151 }
 152
 153 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 154 {
 155     return (char *)env + ri->fieldoffset;
 156 }
 157
 158 uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 159 {
 160     /* Raw read of a coprocessor register (as needed for migration, etc). */
 161     if (ri->type & ARM_CP_CONST) {
 162         return ri->resetvalue;
 163     } else if (ri->raw_readfn) {
 164         return ri->raw_readfn(env, ri);
 165     } else if (ri->readfn) {
 166         return ri->readfn(env, ri);
 167     } else {
 168         return raw_read(env, ri);
 169     }
 170 }
 171
 172 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
 173                              uint64_t v)
 174 {
 175     /* Raw write of a coprocessor register (as needed for migration, etc).
 176      * Note that constant registers are treated as write-ignored; the
 177      * caller should check for success by whether a readback gives the
 178      * value written.
 179      */
 180     if (ri->type & ARM_CP_CONST) {
 181         return;
 182     } else if (ri->raw_writefn) {
 183         ri->raw_writefn(env, ri, v);
 184     } else if (ri->writefn) {
 185         ri->writefn(env, ri, v);
 186     } else {
 187         raw_write(env, ri, v);
 188     }
 189 }
 190
 191 static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
 192 {
 193    /* Return true if the regdef would cause an assertion if you called
 194     * read_raw_cp_reg() or write_raw_cp_reg() on it (ie if it is a
 195     * program bug for it not to have the NO_RAW flag).
 196     * NB that returning false here doesn't necessarily mean that calling
 197     * read/write_raw_cp_reg() is safe, because we can't distinguish "has
 198     * read/write access functions which are safe for raw use" from "has
 199     * read/write access functions which have side effects but has forgotten
 200     * to provide raw access functions".
 201     * The tests here line up with the conditions in read/write_raw_cp_reg()
 202     * and assertions in raw_read()/raw_write().
 203     */
 204     if ((ri->type & ARM_CP_CONST) ||
 205         ri->fieldoffset ||
 206         ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
 207         return false;
 208     }
 209     return true;
 210 }
 211
 212 bool write_cpustate_to_list(ARMCPU *cpu)
 213 {
 214     /* Write the coprocessor state from cpu->env to the (index,value) list. */
 215     int i;
 216     bool ok = true;
 217
 218     for (i = 0; i < cpu->cpreg_array_len; i++) {
 219         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 220         const ARMCPRegInfo *ri;
 221
 222         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 223         if (!ri) {
 224             ok = false;
 225             continue;
 226         }
 227         if (ri->type & ARM_CP_NO_RAW) {
 228             continue;
 229         }
 230         cpu->cpreg_values[i] = read_raw_cp_reg(&cpu->env, ri);
 231     }
 232     return ok;
 233 }
 234
 235 bool write_list_to_cpustate(ARMCPU *cpu)
 236 {
 237     int i;
 238     bool ok = true;
 239
 240     for (i = 0; i < cpu->cpreg_array_len; i++) {
 241         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 242         uint64_t v = cpu->cpreg_values[i];
 243         const ARMCPRegInfo *ri;
 244
 245         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 246         if (!ri) {
 247             ok = false;
 248             continue;
 249         }
 250         if (ri->type & ARM_CP_NO_RAW) {
 251             continue;
 252         }
 253         /* Write value and confirm it reads back as written
 254          * (to catch read-only registers and partially read-only
 255          * registers where the incoming migration value doesn't match)
 256          */
 257         write_raw_cp_reg(&cpu->env, ri, v);
 258         if (read_raw_cp_reg(&cpu->env, ri) != v) {
 259             ok = false;
 260         }
 261     }
 262     return ok;
 263 }
 264
 265 static void add_cpreg_to_list(gpointer key, gpointer opaque)
 266 {
 267     ARMCPU *cpu = opaque;
 268     uint64_t regidx;
 269     const ARMCPRegInfo *ri;
 270
 271     regidx = *(uint32_t *)key;
 272     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 273
 274     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 275         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
 276         /* The value array need not be initialized at this point */
 277         cpu->cpreg_array_len++;
 278     }
 279 }
 280
 281 static void count_cpreg(gpointer key, gpointer opaque)
 282 {
 283     ARMCPU *cpu = opaque;
 284     uint64_t regidx;
 285     const ARMCPRegInfo *ri;
 286
 287     regidx = *(uint32_t *)key;
 288     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 289
 290     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 291         cpu->cpreg_array_len++;
 292     }
 293 }
 294
 295 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
 296 {
 297     uint64_t aidx = cpreg_to_kvm_id(*(uint32_t *)a);
 298     uint64_t bidx = cpreg_to_kvm_id(*(uint32_t *)b);
 299
 300     if (aidx > bidx) {
 301         return 1;
 302     }
 303     if (aidx < bidx) {
 304         return -1;
 305     }
 306     return 0;
 307 }
 308
 309 void init_cpreg_list(ARMCPU *cpu)
 310 {
 311     /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
 312      * Note that we require cpreg_tuples[] to be sorted by key ID.
 313      */
 314     GList *keys;
 315     int arraylen;
 316
 317     keys = g_hash_table_get_keys(cpu->cp_regs);
 318     keys = g_list_sort(keys, cpreg_key_compare);
 319
 320     cpu->cpreg_array_len = 0;
 321
 322     g_list_foreach(keys, count_cpreg, cpu);
 323
 324     arraylen = cpu->cpreg_array_len;
 325     cpu->cpreg_indexes = g_new(uint64_t, arraylen);
 326     cpu->cpreg_values = g_new(uint64_t, arraylen);
 327     cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
 328     cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
 329     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
 330     cpu->cpreg_array_len = 0;
 331
 332     g_list_foreach(keys, add_cpreg_to_list, cpu);
 333
 334     assert(cpu->cpreg_array_len == arraylen);
 335
 336     g_list_free(keys);
 337 }
 338
 339 /*
 340  * Some registers are not accessible if EL3.NS=0 and EL3 is using AArch32 but
 341  * they are accessible when EL3 is using AArch64 regardless of EL3.NS.
 342  *
 343  * access_el3_aa32ns: Used to check AArch32 register views.
 344  * access_el3_aa32ns_aa64any: Used to check both AArch32/64 register views.
 345  */
 346 static CPAccessResult access_el3_aa32ns(CPUARMState *env,
 347                                         const ARMCPRegInfo *ri,
 348                                         bool isread)
 349 {
 350     bool secure = arm_is_secure_below_el3(env);
 351
 352     assert(!arm_el_is_aa64(env, 3));
 353     if (secure) {
 354         return CP_ACCESS_TRAP_UNCATEGORIZED;
 355     }
 356     return CP_ACCESS_OK;
 357 }
 358
 359 static CPAccessResult access_el3_aa32ns_aa64any(CPUARMState *env,
 360                                                 const ARMCPRegInfo *ri,
 361                                                 bool isread)
 362 {
 363     if (!arm_el_is_aa64(env, 3)) {
 364         return access_el3_aa32ns(env, ri, isread);
 365     }
 366     return CP_ACCESS_OK;
 367 }
 368
 369 /* Some secure-only AArch32 registers trap to EL3 if used from
 370  * Secure EL1 (but are just ordinary UNDEF in other non-EL3 contexts).
 371  * Note that an access from Secure EL1 can only happen if EL3 is AArch64.
 372  * We assume that the .access field is set to PL1_RW.
 373  */
 374 static CPAccessResult access_trap_aa32s_el1(CPUARMState *env,
 375                                             const ARMCPRegInfo *ri,
 376                                             bool isread)
 377 {
 378     if (arm_current_el(env) == 3) {
 379         return CP_ACCESS_OK;
 380     }
 381     if (arm_is_secure_below_el3(env)) {
 382         return CP_ACCESS_TRAP_EL3;
 383     }
 384     /* This will be EL1 NS and EL2 NS, which just UNDEF */
 385     return CP_ACCESS_TRAP_UNCATEGORIZED;
 386 }
 387
 388 /* Check for traps to "powerdown debug" registers, which are controlled
 389  * by MDCR.TDOSA
 390  */
 391 static CPAccessResult access_tdosa(CPUARMState *env, const ARMCPRegInfo *ri,
 392                                    bool isread)
 393 {
 394     int el = arm_current_el(env);
 395
 396     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDOSA)
 397         && !arm_is_secure_below_el3(env)) {
 398         return CP_ACCESS_TRAP_EL2;
 399     }
 400     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDOSA)) {
 401         return CP_ACCESS_TRAP_EL3;
 402     }
 403     return CP_ACCESS_OK;
 404 }
 405
 406 /* Check for traps to "debug ROM" registers, which are controlled
 407  * by MDCR_EL2.TDRA for EL2 but by the more general MDCR_EL3.TDA for EL3.
 408  */
 409 static CPAccessResult access_tdra(CPUARMState *env, const ARMCPRegInfo *ri,
 410                                   bool isread)
 411 {
 412     int el = arm_current_el(env);
 413
 414     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDRA)
 415         && !arm_is_secure_below_el3(env)) {
 416         return CP_ACCESS_TRAP_EL2;
 417     }
 418     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
 419         return CP_ACCESS_TRAP_EL3;
 420     }
 421     return CP_ACCESS_OK;
 422 }
 423
 424 /* Check for traps to general debug registers, which are controlled
 425  * by MDCR_EL2.TDA for EL2 and MDCR_EL3.TDA for EL3.
 426  */
 427 static CPAccessResult access_tda(CPUARMState *env, const ARMCPRegInfo *ri,
 428                                   bool isread)
 429 {
 430     int el = arm_current_el(env);
 431
 432     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDA)
 433         && !arm_is_secure_below_el3(env)) {
 434         return CP_ACCESS_TRAP_EL2;
 435     }
 436     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
 437         return CP_ACCESS_TRAP_EL3;
 438     }
 439     return CP_ACCESS_OK;
 440 }
 441
 442 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 443 {
 444     ARMCPU *cpu = arm_env_get_cpu(env);
 445
 446     raw_write(env, ri, value);
 447     tlb_flush(CPU(cpu), 1); /* Flush TLB as domain not tracked in TLB */
 448 }
 449
 450 static void fcse_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 451 {
 452     ARMCPU *cpu = arm_env_get_cpu(env);
 453
 454     if (raw_read(env, ri) != value) {
 455         /* Unlike real hardware the qemu TLB uses virtual addresses,
 456          * not modified virtual addresses, so this causes a TLB flush.
 457          */
 458         tlb_flush(CPU(cpu), 1);
 459         raw_write(env, ri, value);
 460     }
 461 }
 462
 463 static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 464                              uint64_t value)
 465 {
 466     ARMCPU *cpu = arm_env_get_cpu(env);
 467
 468     if (raw_read(env, ri) != value && !arm_feature(env, ARM_FEATURE_MPU)
 469         && !extended_addresses_enabled(env)) {
 470         /* For VMSA (when not using the LPAE long descriptor page table
 471          * format) this register includes the ASID, so do a TLB flush.
 472          * For PMSA it is purely a process ID and no action is needed.
 473          */
 474         tlb_flush(CPU(cpu), 1);
 475     }
 476     raw_write(env, ri, value);
 477 }
 478
 479 static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
 480                           uint64_t value)
 481 {
 482     /* Invalidate all (TLBIALL) */
 483     ARMCPU *cpu = arm_env_get_cpu(env);
 484
 485     tlb_flush(CPU(cpu), 1);
 486 }
 487
 488 static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri,
 489                           uint64_t value)
 490 {
 491     /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */
 492     ARMCPU *cpu = arm_env_get_cpu(env);
 493
 494     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 495 }
 496
 497 static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri,
 498                            uint64_t value)
 499 {
 500     /* Invalidate by ASID (TLBIASID) */
 501     ARMCPU *cpu = arm_env_get_cpu(env);
 502
 503     tlb_flush(CPU(cpu), value == 0);
 504 }
 505
 506 static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
 507                            uint64_t value)
 508 {
 509     /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */
 510     ARMCPU *cpu = arm_env_get_cpu(env);
 511
 512     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 513 }
 514
 515 /* IS variants of TLB operations must affect all cores */
 516 static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 517                              uint64_t value)
 518 {
 519     CPUState *other_cs;
 520
 521     CPU_FOREACH(other_cs) {
 522         tlb_flush(other_cs, 1);
 523     }
 524 }
 525
 526 static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 527                              uint64_t value)
 528 {
 529     CPUState *other_cs;
 530
 531     CPU_FOREACH(other_cs) {
 532         tlb_flush(other_cs, value == 0);
 533     }
 534 }
 535
 536 static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 537                              uint64_t value)
 538 {
 539     CPUState *other_cs;
 540
 541     CPU_FOREACH(other_cs) {
 542         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 543     }
 544 }
 545
 546 static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 547                              uint64_t value)
 548 {
 549     CPUState *other_cs;
 550
 551     CPU_FOREACH(other_cs) {
 552         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 553     }
 554 }
 555
 556 static const ARMCPRegInfo cp_reginfo[] = {
 557     /* Define the secure and non-secure FCSE identifier CP registers
 558      * separately because there is no secure bank in V8 (no _EL3).  This allows
 559      * the secure register to be properly reset and migrated. There is also no
 560      * v8 EL1 version of the register so the non-secure instance stands alone.
 561      */
 562     { .name = "FCSEIDR(NS)",
 563       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 564       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 565       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
 566       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 567     { .name = "FCSEIDR(S)",
 568       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 569       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 570       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
 571       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 572     /* Define the secure and non-secure context identifier CP registers
 573      * separately because there is no secure bank in V8 (no _EL3).  This allows
 574      * the secure register to be properly reset and migrated.  In the
 575      * non-secure case, the 32-bit register will have reset and migration
 576      * disabled during registration as it is handled by the 64-bit instance.
 577      */
 578     { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
 579       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 580       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 581       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
 582       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 583     { .name = "CONTEXTIDR(S)", .state = ARM_CP_STATE_AA32,
 584       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 585       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 586       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
 587       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 588     REGINFO_SENTINEL
 589 };
 590
 591 static const ARMCPRegInfo not_v8_cp_reginfo[] = {
 592     /* NB: Some of these registers exist in v8 but with more precise
 593      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
 594      */
 595     /* MMU Domain access control / MPU write buffer control */
 596     { .name = "DACR",
 597       .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
 598       .access = PL1_RW, .resetvalue = 0,
 599       .writefn = dacr_write, .raw_writefn = raw_write,
 600       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
 601                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
 602     /* ARMv7 allocates a range of implementation defined TLB LOCKDOWN regs.
 603      * For v6 and v5, these mappings are overly broad.
 604      */
 605     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 0,
 606       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 607     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 1,
 608       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 609     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 4,
 610       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 611     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 8,
 612       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 613     /* Cache maintenance ops; some of this space may be overridden later. */
 614     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
 615       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
 616       .type = ARM_CP_NOP | ARM_CP_OVERRIDE },
 617     REGINFO_SENTINEL
 618 };
 619
 620 static const ARMCPRegInfo not_v6_cp_reginfo[] = {
 621     /* Not all pre-v6 cores implemented this WFI, so this is slightly
 622      * over-broad.
 623      */
 624     { .name = "WFI_v5", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = 2,
 625       .access = PL1_W, .type = ARM_CP_WFI },
 626     REGINFO_SENTINEL
 627 };
 628
 629 static const ARMCPRegInfo not_v7_cp_reginfo[] = {
 630     /* Standard v6 WFI (also used in some pre-v6 cores); not in v7 (which
 631      * is UNPREDICTABLE; we choose to NOP as most implementations do).
 632      */
 633     { .name = "WFI_v6", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 634       .access = PL1_W, .type = ARM_CP_WFI },
 635     /* L1 cache lockdown. Not architectural in v6 and earlier but in practice
 636      * implemented in 926, 946, 1026, 1136, 1176 and 11MPCore. StrongARM and
 637      * OMAPCP will override this space.
 638      */
 639     { .name = "DLOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 0,
 640       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_data),
 641       .resetvalue = 0 },
 642     { .name = "ILOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 1,
 643       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_insn),
 644       .resetvalue = 0 },
 645     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
 646     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
 647       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
 648       .resetvalue = 0 },
 649     /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
 650      * implementing it as RAZ means the "debug architecture version" bits
 651      * will read as a reserved value, which should cause Linux to not try
 652      * to use the debug hardware.
 653      */
 654     { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
 655       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
 656     /* MMU TLB control. Note that the wildcarding means we cover not just
 657      * the unified TLB ops but also the dside/iside/inner-shareable variants.
 658      */
 659     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
 660       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
 661       .type = ARM_CP_NO_RAW },
 662     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
 663       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
 664       .type = ARM_CP_NO_RAW },
 665     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
 666       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
 667       .type = ARM_CP_NO_RAW },
 668     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
 669       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
 670       .type = ARM_CP_NO_RAW },
 671     { .name = "PRRR", .cp = 15, .crn = 10, .crm = 2,
 672       .opc1 = 0, .opc2 = 0, .access = PL1_RW, .type = ARM_CP_NOP },
 673     { .name = "NMRR", .cp = 15, .crn = 10, .crm = 2,
 674       .opc1 = 0, .opc2 = 1, .access = PL1_RW, .type = ARM_CP_NOP },
 675     REGINFO_SENTINEL
 676 };
 677
 678 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 679                         uint64_t value)
 680 {
 681     uint32_t mask = 0;
 682
 683     /* In ARMv8 most bits of CPACR_EL1 are RES0. */
 684     if (!arm_feature(env, ARM_FEATURE_V8)) {
 685         /* ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
 686          * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
 687          * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
 688          */
 689         if (arm_feature(env, ARM_FEATURE_VFP)) {
 690             /* VFP coprocessor: cp10 & cp11 [23:20] */
 691             mask |= (1 << 31) | (1 << 30) | (0xf << 20);
 692
 693             if (!arm_feature(env, ARM_FEATURE_NEON)) {
 694                 /* ASEDIS [31] bit is RAO/WI */
 695                 value |= (1 << 31);
 696             }
 697
 698             /* VFPv3 and upwards with NEON implement 32 double precision
 699              * registers (D0-D31).
 700              */
 701             if (!arm_feature(env, ARM_FEATURE_NEON) ||
 702                     !arm_feature(env, ARM_FEATURE_VFP3)) {
 703                 /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
 704                 value |= (1 << 30);
 705             }
 706         }
 707         value &= mask;
 708     }
 709     env->cp15.cpacr_el1 = value;
 710 }
 711
 712 static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
 713                                    bool isread)
 714 {
 715     if (arm_feature(env, ARM_FEATURE_V8)) {
 716         /* Check if CPACR accesses are to be trapped to EL2 */
 717         if (arm_current_el(env) == 1 &&
 718             (env->cp15.cptr_el[2] & CPTR_TCPAC) && !arm_is_secure(env)) {
 719             return CP_ACCESS_TRAP_EL2;
 720         /* Check if CPACR accesses are to be trapped to EL3 */
 721         } else if (arm_current_el(env) < 3 &&
 722                    (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
 723             return CP_ACCESS_TRAP_EL3;
 724         }
 725     }
 726
 727     return CP_ACCESS_OK;
 728 }
 729
 730 static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri,
 731                                   bool isread)
 732 {
 733     /* Check if CPTR accesses are set to trap to EL3 */
 734     if (arm_current_el(env) == 2 && (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
 735         return CP_ACCESS_TRAP_EL3;
 736     }
 737
 738     return CP_ACCESS_OK;
 739 }
 740
 741 static const ARMCPRegInfo v6_cp_reginfo[] = {
 742     /* prefetch by MVA in v6, NOP in v7 */
 743     { .name = "MVA_prefetch",
 744       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
 745       .access = PL1_W, .type = ARM_CP_NOP },
 746     /* We need to break the TB after ISB to execute self-modifying code
 747      * correctly and also to take any pending interrupts immediately.
 748      * So use arm_cp_write_ignore() function instead of ARM_CP_NOP flag.
 749      */
 750     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
 751       .access = PL0_W, .type = ARM_CP_NO_RAW, .writefn = arm_cp_write_ignore },
 752     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
 753       .access = PL0_W, .type = ARM_CP_NOP },
 754     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
 755       .access = PL0_W, .type = ARM_CP_NOP },
 756     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
 757       .access = PL1_RW,
 758       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
 759                              offsetof(CPUARMState, cp15.ifar_ns) },
 760       .resetvalue = 0, },
 761     /* Watchpoint Fault Address Register : should actually only be present
 762      * for 1136, 1176, 11MPCore.
 763      */
 764     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
 765       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
 766     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
 767       .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2, .accessfn = cpacr_access,
 768       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.cpacr_el1),
 769       .resetvalue = 0, .writefn = cpacr_write },
 770     REGINFO_SENTINEL
 771 };
 772
 773 static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri,
 774                                    bool isread)
 775 {
 776     /* Performance monitor registers user accessibility is controlled
 777      * by PMUSERENR.
 778      */
 779     if (arm_current_el(env) == 0 && !env->cp15.c9_pmuserenr) {
 780         return CP_ACCESS_TRAP;
 781     }
 782     return CP_ACCESS_OK;
 783 }
 784
 785 #ifndef CONFIG_USER_ONLY
 786
 787 static inline bool arm_ccnt_enabled(CPUARMState *env)
 788 {
 789     /* This does not support checking PMCCFILTR_EL0 register */
 790
 791     if (!(env->cp15.c9_pmcr & PMCRE)) {
 792         return false;
 793     }
 794
 795     return true;
 796 }
 797
 798 void pmccntr_sync(CPUARMState *env)
 799 {
 800     uint64_t temp_ticks;
 801
 802     temp_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
 803                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 804
 805     if (env->cp15.c9_pmcr & PMCRD) {
 806         /* Increment once every 64 processor clock cycles */
 807         temp_ticks /= 64;
 808     }
 809
 810     if (arm_ccnt_enabled(env)) {
 811         env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
 812     }
 813 }
 814
 815 static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 816                        uint64_t value)
 817 {
 818     pmccntr_sync(env);
 819
 820     if (value & PMCRC) {
 821         /* The counter has been reset */
 822         env->cp15.c15_ccnt = 0;
 823     }
 824
 825     /* only the DP, X, D and E bits are writable */
 826     env->cp15.c9_pmcr &= ~0x39;
 827     env->cp15.c9_pmcr |= (value & 0x39);
 828
 829     pmccntr_sync(env);
 830 }
 831
 832 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 833 {
 834     uint64_t total_ticks;
 835
 836     if (!arm_ccnt_enabled(env)) {
 837         /* Counter is disabled, do not change value */
 838         return env->cp15.c15_ccnt;
 839     }
 840
 841     total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
 842                            ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 843
 844     if (env->cp15.c9_pmcr & PMCRD) {
 845         /* Increment once every 64 processor clock cycles */
 846         total_ticks /= 64;
 847     }
 848     return total_ticks - env->cp15.c15_ccnt;
 849 }
 850
 851 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 852                         uint64_t value)
 853 {
 854     uint64_t total_ticks;
 855
 856     if (!arm_ccnt_enabled(env)) {
 857         /* Counter is disabled, set the absolute value */
 858         env->cp15.c15_ccnt = value;
 859         return;
 860     }
 861
 862     total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
 863                            ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 864
 865     if (env->cp15.c9_pmcr & PMCRD) {
 866         /* Increment once every 64 processor clock cycles */
 867         total_ticks /= 64;
 868     }
 869     env->cp15.c15_ccnt = total_ticks - value;
 870 }
 871
 872 static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
 873                             uint64_t value)
 874 {
 875     uint64_t cur_val = pmccntr_read(env, NULL);
 876
 877     pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
 878 }
 879
 880 #else /* CONFIG_USER_ONLY */
 881
 882 void pmccntr_sync(CPUARMState *env)
 883 {
 884 }
 885
 886 #endif
 887
 888 static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 889                             uint64_t value)
 890 {
 891     pmccntr_sync(env);
 892     env->cp15.pmccfiltr_el0 = value & 0x7E000000;
 893     pmccntr_sync(env);
 894 }
 895
 896 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 897                             uint64_t value)
 898 {
 899     value &= (1 << 31);
 900     env->cp15.c9_pmcnten |= value;
 901 }
 902
 903 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 904                              uint64_t value)
 905 {
 906     value &= (1 << 31);
 907     env->cp15.c9_pmcnten &= ~value;
 908 }
 909
 910 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 911                          uint64_t value)
 912 {
 913     env->cp15.c9_pmovsr &= ~value;
 914 }
 915
 916 static void pmxevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
 917                              uint64_t value)
 918 {
 919     env->cp15.c9_pmxevtyper = value & 0xff;
 920 }
 921
 922 static void pmuserenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 923                             uint64_t value)
 924 {
 925     env->cp15.c9_pmuserenr = value & 1;
 926 }
 927
 928 static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 929                              uint64_t value)
 930 {
 931     /* We have no event counters so only the C bit can be changed */
 932     value &= (1 << 31);
 933     env->cp15.c9_pminten |= value;
 934 }
 935
 936 static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 937                              uint64_t value)
 938 {
 939     value &= (1 << 31);
 940     env->cp15.c9_pminten &= ~value;
 941 }
 942
 943 static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
 944                        uint64_t value)
 945 {
 946     /* Note that even though the AArch64 view of this register has bits
 947      * [10:0] all RES0 we can only mask the bottom 5, to comply with the
 948      * architectural requirements for bits which are RES0 only in some
 949      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
 950      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
 951      */
 952     raw_write(env, ri, value & ~0x1FULL);
 953 }
 954
 955 static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 956 {
 957     /* We only mask off bits that are RES0 both for AArch64 and AArch32.
 958      * For bits that vary between AArch32/64, code needs to check the
 959      * current execution mode before directly using the feature bit.
 960      */
 961     uint32_t valid_mask = SCR_AARCH64_MASK | SCR_AARCH32_MASK;
 962
 963     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 964         valid_mask &= ~SCR_HCE;
 965
 966         /* On ARMv7, SMD (or SCD as it is called in v7) is only
 967          * supported if EL2 exists. The bit is UNK/SBZP when
 968          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
 969          * when EL2 is unavailable.
 970          * On ARMv8, this bit is always available.
 971          */
 972         if (arm_feature(env, ARM_FEATURE_V7) &&
 973             !arm_feature(env, ARM_FEATURE_V8)) {
 974             valid_mask &= ~SCR_SMD;
 975         }
 976     }
 977
 978     /* Clear all-context RES0 bits.  */
 979     value &= valid_mask;
 980     raw_write(env, ri, value);
 981 }
 982
 983 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 984 {
 985     ARMCPU *cpu = arm_env_get_cpu(env);
 986
 987     /* Acquire the CSSELR index from the bank corresponding to the CCSIDR
 988      * bank
 989      */
 990     uint32_t index = A32_BANKED_REG_GET(env, csselr,
 991                                         ri->secure & ARM_CP_SECSTATE_S);
 992
 993     return cpu->ccsidr[index];
 994 }
 995
 996 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 997                          uint64_t value)
 998 {
 999     raw_write(env, ri, value & 0xf);
1000 }
1001
1002 static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1003 {
1004     CPUState *cs = ENV_GET_CPU(env);
1005     uint64_t ret = 0;
1006
1007     if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
1008         ret |= CPSR_I;
1009     }
1010     if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
1011         ret |= CPSR_F;
1012     }
1013     /* External aborts are not possible in QEMU so A bit is always clear */
1014     return ret;
1015 }
1016
1017 static const ARMCPRegInfo v7_cp_reginfo[] = {
1018     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
1019     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
1020       .access = PL1_W, .type = ARM_CP_NOP },
1021     /* Performance monitors are implementation defined in v7,
1022      * but with an ARM recommended set of registers, which we
1023      * follow (although we don't actually implement any counters)
1024      *
1025      * Performance registers fall into three categories:
1026      *  (a) always UNDEF in PL0, RW in PL1 (PMINTENSET, PMINTENCLR)
1027      *  (b) RO in PL0 (ie UNDEF on write), RW in PL1 (PMUSERENR)
1028      *  (c) UNDEF in PL0 if PMUSERENR.EN==0, otherwise accessible (all others)
1029      * For the cases controlled by PMUSERENR we must set .access to PL0_RW
1030      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
1031      */
1032     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
1033       .access = PL0_RW, .type = ARM_CP_ALIAS,
1034       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
1035       .writefn = pmcntenset_write,
1036       .accessfn = pmreg_access,
1037       .raw_writefn = raw_write },
1038     { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64,
1039       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
1040       .access = PL0_RW, .accessfn = pmreg_access,
1041       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
1042       .writefn = pmcntenset_write, .raw_writefn = raw_write },
1043     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
1044       .access = PL0_RW,
1045       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
1046       .accessfn = pmreg_access,
1047       .writefn = pmcntenclr_write,
1048       .type = ARM_CP_ALIAS },
1049     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
1050       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
1051       .access = PL0_RW, .accessfn = pmreg_access,
1052       .type = ARM_CP_ALIAS,
1053       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
1054       .writefn = pmcntenclr_write },
1055     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
1056       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
1057       .accessfn = pmreg_access,
1058       .writefn = pmovsr_write,
1059       .raw_writefn = raw_write },
1060     { .name = "PMOVSCLR_EL0", .state = ARM_CP_STATE_AA64,
1061       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 3,
1062       .access = PL0_RW, .accessfn = pmreg_access,
1063       .type = ARM_CP_ALIAS,
1064       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
1065       .writefn = pmovsr_write,
1066       .raw_writefn = raw_write },
1067     /* Unimplemented so WI. */
1068     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
1069       .access = PL0_W, .accessfn = pmreg_access, .type = ARM_CP_NOP },
1070     /* Since we don't implement any events, writing to PMSELR is UNPREDICTABLE.
1071      * We choose to RAZ/WI.
1072      */
1073     { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5,
1074       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
1075       .accessfn = pmreg_access },
1076 #ifndef CONFIG_USER_ONLY
1077     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
1078       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_IO,
1079       .readfn = pmccntr_read, .writefn = pmccntr_write32,
1080       .accessfn = pmreg_access },
1081     { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
1082       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
1083       .access = PL0_RW, .accessfn = pmreg_access,
1084       .type = ARM_CP_IO,
1085       .readfn = pmccntr_read, .writefn = pmccntr_write, },
1086 #endif
1087     { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
1088       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
1089       .writefn = pmccfiltr_write,
1090       .access = PL0_RW, .accessfn = pmreg_access,
1091       .type = ARM_CP_IO,
1092       .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
1093       .resetvalue = 0, },
1094     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
1095       .access = PL0_RW,
1096       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmxevtyper),
1097       .accessfn = pmreg_access, .writefn = pmxevtyper_write,
1098       .raw_writefn = raw_write },
1099     /* Unimplemented, RAZ/WI. */
1100     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
1101       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
1102       .accessfn = pmreg_access },
1103     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
1104       .access = PL0_R | PL1_RW,
1105       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
1106       .resetvalue = 0,
1107       .writefn = pmuserenr_write, .raw_writefn = raw_write },
1108     { .name = "PMUSERENR_EL0", .state = ARM_CP_STATE_AA64,
1109       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 0,
1110       .access = PL0_R | PL1_RW, .type = ARM_CP_ALIAS,
1111       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
1112       .resetvalue = 0,
1113       .writefn = pmuserenr_write, .raw_writefn = raw_write },
1114     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
1115       .access = PL1_RW,
1116       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
1117       .resetvalue = 0,
1118       .writefn = pmintenset_write, .raw_writefn = raw_write },
1119     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
1120       .access = PL1_RW, .type = ARM_CP_ALIAS,
1121       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
1122       .writefn = pmintenclr_write, },
1123     { .name = "PMINTENCLR_EL1", .state = ARM_CP_STATE_AA64,
1124       .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 2,
1125       .access = PL1_RW, .type = ARM_CP_ALIAS,
1126       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
1127       .writefn = pmintenclr_write },
1128     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
1129       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
1130       .access = PL1_RW, .writefn = vbar_write,
1131       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
1132                              offsetof(CPUARMState, cp15.vbar_ns) },
1133       .resetvalue = 0 },
1134     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
1135       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
1136       .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_RAW },
1137     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
1138       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
1139       .access = PL1_RW, .writefn = csselr_write, .resetvalue = 0,
1140       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
1141                              offsetof(CPUARMState, cp15.csselr_ns) } },
1142     /* Auxiliary ID register: this actually has an IMPDEF value but for now
1143      * just RAZ for all cores:
1144      */
1145     { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
1146       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
1147       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
1148     /* Auxiliary fault status registers: these also are IMPDEF, and we
1149      * choose to RAZ/WI for all cores.
1150      */
1151     { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
1152       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
1153       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
1154     { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
1155       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
1156       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
1157     /* MAIR can just read-as-written because we don't implement caches
1158      * and so don't need to care about memory attributes.
1159      */
1160     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
1161       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
1162       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
1163       .resetvalue = 0 },
1164     { .name = "MAIR_EL3", .state = ARM_CP_STATE_AA64,
1165       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 2, .opc2 = 0,
1166       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[3]),
1167       .resetvalue = 0 },
1168     /* For non-long-descriptor page tables these are PRRR and NMRR;
1169      * regardless they still act as reads-as-written for QEMU.
1170      */
1171      /* MAIR0/1 are defined separately from their 64-bit counterpart which
1172       * allows them to assign the correct fieldoffset based on the endianness
1173       * handled in the field definitions.
1174       */
1175     { .name = "MAIR0", .state = ARM_CP_STATE_AA32,
1176       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0, .access = PL1_RW,
1177       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
1178                              offsetof(CPUARMState, cp15.mair0_ns) },
1179       .resetfn = arm_cp_reset_ignore },
1180     { .name = "MAIR1", .state = ARM_CP_STATE_AA32,
1181       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1, .access = PL1_RW,
1182       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
1183                              offsetof(CPUARMState, cp15.mair1_ns) },
1184       .resetfn = arm_cp_reset_ignore },
1185     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
1186       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
1187       .type = ARM_CP_NO_RAW, .access = PL1_R, .readfn = isr_read },
1188     /* 32 bit ITLB invalidates */
1189     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
1190       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1191     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
1192       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1193     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
1194       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1195     /* 32 bit DTLB invalidates */
1196     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
1197       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1198     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
1199       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1200     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
1201       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1202     /* 32 bit TLB invalidates */
1203     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
1204       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1205     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
1206       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1207     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
1208       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1209     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
1210       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
1211     REGINFO_SENTINEL
1212 };
1213
1214 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
1215     /* 32 bit TLB invalidates, Inner Shareable */
1216     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
1217       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_is_write },
1218     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
1219       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
1220     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
1221       .type = ARM_CP_NO_RAW, .access = PL1_W,
1222       .writefn = tlbiasid_is_write },
1223     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
1224       .type = ARM_CP_NO_RAW, .access = PL1_W,
1225       .writefn = tlbimvaa_is_write },
1226     REGINFO_SENTINEL
1227 };
1228
1229 static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1230                         uint64_t value)
1231 {
1232     value &= 1;
1233     env->teecr = value;
1234 }
1235
1236 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri,
1237                                     bool isread)
1238 {
1239     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
1240         return CP_ACCESS_TRAP;
1241     }
1242     return CP_ACCESS_OK;
1243 }
1244
1245 static const ARMCPRegInfo t2ee_cp_reginfo[] = {
1246     { .name = "TEECR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 6, .opc2 = 0,
1247       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, teecr),
1248       .resetvalue = 0,
1249       .writefn = teecr_write },
1250     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
1251       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
1252       .accessfn = teehbr_access, .resetvalue = 0 },
1253     REGINFO_SENTINEL
1254 };
1255
1256 static const ARMCPRegInfo v6k_cp_reginfo[] = {
1257     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
1258       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
1259       .access = PL0_RW,
1260       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
1261     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
1262       .access = PL0_RW,
1263       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
1264                              offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
1265       .resetfn = arm_cp_reset_ignore },
1266     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
1267       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
1268       .access = PL0_R|PL1_W,
1269       .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
1270       .resetvalue = 0},
1271     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
1272       .access = PL0_R|PL1_W,
1273       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
1274                              offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
1275       .resetfn = arm_cp_reset_ignore },
1276     { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
1277       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
1278       .access = PL1_RW,
1279       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
1280     { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
1281       .access = PL1_RW,
1282       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
1283                              offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
1284       .resetvalue = 0 },
1285     REGINFO_SENTINEL
1286 };
1287
1288 #ifndef CONFIG_USER_ONLY
1289
1290 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri,
1291                                        bool isread)
1292 {
1293     /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero.
1294      * Writable only at the highest implemented exception level.
1295      */
1296     int el = arm_current_el(env);
1297
1298     switch (el) {
1299     case 0:
1300         if (!extract32(env->cp15.c14_cntkctl, 0, 2)) {
1301             return CP_ACCESS_TRAP;
1302         }
1303         break;
1304     case 1:
1305         if (!isread && ri->state == ARM_CP_STATE_AA32 &&
1306             arm_is_secure_below_el3(env)) {
1307             /* Accesses from 32-bit Secure EL1 UNDEF (*not* trap to EL3!) */
1308             return CP_ACCESS_TRAP_UNCATEGORIZED;
1309         }
1310         break;
1311     case 2:
1312     case 3:
1313         break;
1314     }
1315
1316     if (!isread && el < arm_highest_el(env)) {
1317         return CP_ACCESS_TRAP_UNCATEGORIZED;
1318     }
1319
1320     return CP_ACCESS_OK;
1321 }
1322
1323 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx,
1324                                         bool isread)
1325 {
1326     unsigned int cur_el = arm_current_el(env);
1327     bool secure = arm_is_secure(env);
1328
1329     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
1330     if (cur_el == 0 &&
1331         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
1332         return CP_ACCESS_TRAP;
1333     }
1334
1335     if (arm_feature(env, ARM_FEATURE_EL2) &&
1336         timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
1337         !extract32(env->cp15.cnthctl_el2, 0, 1)) {
1338         return CP_ACCESS_TRAP_EL2;
1339     }
1340     return CP_ACCESS_OK;
1341 }
1342
1343 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx,
1344                                       bool isread)
1345 {
1346     unsigned int cur_el = arm_current_el(env);
1347     bool secure = arm_is_secure(env);
1348
1349     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
1350      * EL0[PV]TEN is zero.
1351      */
1352     if (cur_el == 0 &&
1353         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
1354         return CP_ACCESS_TRAP;
1355     }
1356
1357     if (arm_feature(env, ARM_FEATURE_EL2) &&
1358         timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
1359         !extract32(env->cp15.cnthctl_el2, 1, 1)) {
1360         return CP_ACCESS_TRAP_EL2;
1361     }
1362     return CP_ACCESS_OK;
1363 }
1364
1365 static CPAccessResult gt_pct_access(CPUARMState *env,
1366                                     const ARMCPRegInfo *ri,
1367                                     bool isread)
1368 {
1369     return gt_counter_access(env, GTIMER_PHYS, isread);
1370 }
1371
1372 static CPAccessResult gt_vct_access(CPUARMState *env,
1373                                     const ARMCPRegInfo *ri,
1374                                     bool isread)
1375 {
1376     return gt_counter_access(env, GTIMER_VIRT, isread);
1377 }
1378
1379 static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
1380                                        bool isread)
1381 {
1382     return gt_timer_access(env, GTIMER_PHYS, isread);
1383 }
1384
1385 static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
1386                                        bool isread)
1387 {
1388     return gt_timer_access(env, GTIMER_VIRT, isread);
1389 }
1390
1391 static CPAccessResult gt_stimer_access(CPUARMState *env,
1392                                        const ARMCPRegInfo *ri,
1393                                        bool isread)
1394 {
1395     /* The AArch64 register view of the secure physical timer is
1396      * always accessible from EL3, and configurably accessible from
1397      * Secure EL1.
1398      */
1399     switch (arm_current_el(env)) {
1400     case 1:
1401         if (!arm_is_secure(env)) {
1402             return CP_ACCESS_TRAP;
1403         }
1404         if (!(env->cp15.scr_el3 & SCR_ST)) {
1405             return CP_ACCESS_TRAP_EL3;
1406         }
1407         return CP_ACCESS_OK;
1408     case 0:
1409     case 2:
1410         return CP_ACCESS_TRAP;
1411     case 3:
1412         return CP_ACCESS_OK;
1413     default:
1414         g_assert_not_reached();
1415     }
1416 }
1417
1418 static uint64_t gt_get_countervalue(CPUARMState *env)
1419 {
1420     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / GTIMER_SCALE;
1421 }
1422
1423 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
1424 {
1425     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
1426
1427     if (gt->ctl & 1) {
1428         /* Timer enabled: calculate and set current ISTATUS, irq, and
1429          * reset timer to when ISTATUS next has to change
1430          */
1431         uint64_t offset = timeridx == GTIMER_VIRT ?
1432                                       cpu->env.cp15.cntvoff_el2 : 0;
1433         uint64_t count = gt_get_countervalue(&cpu->env);
1434         /* Note that this must be unsigned 64 bit arithmetic: */
1435         int istatus = count - offset >= gt->cval;
1436         uint64_t nexttick;
1437
1438         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
1439         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1440                      (istatus && !(gt->ctl & 2)));
1441         if (istatus) {
1442             /* Next transition is when count rolls back over to zero */
1443             nexttick = UINT64_MAX;
1444         } else {
1445             /* Next transition is when we hit cval */
1446             nexttick = gt->cval + offset;
1447         }
1448         /* Note that the desired next expiry time might be beyond the
1449          * signed-64-bit range of a QEMUTimer -- in this case we just
1450          * set the timer for as far in the future as possible. When the
1451          * timer expires we will reset the timer for any remaining period.
1452          */
1453         if (nexttick > INT64_MAX / GTIMER_SCALE) {
1454             nexttick = INT64_MAX / GTIMER_SCALE;
1455         }
1456         timer_mod(cpu->gt_timer[timeridx], nexttick);
1457     } else {
1458         /* Timer disabled: ISTATUS and timer output always clear */
1459         gt->ctl &= ~4;
1460         qemu_set_irq(cpu->gt_timer_outputs[timeridx], 0);
1461         timer_del(cpu->gt_timer[timeridx]);
1462     }
1463 }
1464
1465 static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
1466                            int timeridx)
1467 {
1468     ARMCPU *cpu = arm_env_get_cpu(env);
1469
1470     timer_del(cpu->gt_timer[timeridx]);
1471 }
1472
1473 static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
1474 {
1475     return gt_get_countervalue(env);
1476 }
1477
1478 static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
1479 {
1480     return gt_get_countervalue(env) - env->cp15.cntvoff_el2;
1481 }
1482
1483 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1484                           int timeridx,
1485                           uint64_t value)
1486 {
1487     env->cp15.c14_timer[timeridx].cval = value;
1488     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1489 }
1490
1491 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
1492                              int timeridx)
1493 {
1494     uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
1495
1496     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
1497                       (gt_get_countervalue(env) - offset));
1498 }
1499
1500 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1501                           int timeridx,
1502                           uint64_t value)
1503 {
1504     uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
1505
1506     env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset +
1507                                          sextract64(value, 0, 32);
1508     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1509 }
1510
1511 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1512                          int timeridx,
1513                          uint64_t value)
1514 {
1515     ARMCPU *cpu = arm_env_get_cpu(env);
1516     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
1517
1518     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
1519     if ((oldval ^ value) & 1) {
1520         /* Enable toggled */
1521         gt_recalc_timer(cpu, timeridx);
1522     } else if ((oldval ^ value) & 2) {
1523         /* IMASK toggled: don't need to recalculate,
1524          * just set the interrupt line based on ISTATUS
1525          */
1526         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1527                      (oldval & 4) && !(value & 2));
1528     }
1529 }
1530
1531 static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1532 {
1533     gt_timer_reset(env, ri, GTIMER_PHYS);
1534 }
1535
1536 static void gt_phys_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1537                                uint64_t value)
1538 {
1539     gt_cval_write(env, ri, GTIMER_PHYS, value);
1540 }
1541
1542 static uint64_t gt_phys_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1543 {
1544     return gt_tval_read(env, ri, GTIMER_PHYS);
1545 }
1546
1547 static void gt_phys_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1548                                uint64_t value)
1549 {
1550     gt_tval_write(env, ri, GTIMER_PHYS, value);
1551 }
1552
1553 static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1554                               uint64_t value)
1555 {
1556     gt_ctl_write(env, ri, GTIMER_PHYS, value);
1557 }
1558
1559 static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1560 {
1561     gt_timer_reset(env, ri, GTIMER_VIRT);
1562 }
1563
1564 static void gt_virt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1565                                uint64_t value)
1566 {
1567     gt_cval_write(env, ri, GTIMER_VIRT, value);
1568 }
1569
1570 static uint64_t gt_virt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1571 {
1572     return gt_tval_read(env, ri, GTIMER_VIRT);
1573 }
1574
1575 static void gt_virt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1576                                uint64_t value)
1577 {
1578     gt_tval_write(env, ri, GTIMER_VIRT, value);
1579 }
1580
1581 static void gt_virt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1582                               uint64_t value)
1583 {
1584     gt_ctl_write(env, ri, GTIMER_VIRT, value);
1585 }
1586
1587 static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri,
1588                               uint64_t value)
1589 {
1590     ARMCPU *cpu = arm_env_get_cpu(env);
1591
1592     raw_write(env, ri, value);
1593     gt_recalc_timer(cpu, GTIMER_VIRT);
1594 }
1595
1596 static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1597 {
1598     gt_timer_reset(env, ri, GTIMER_HYP);
1599 }
1600
1601 static void gt_hyp_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1602                               uint64_t value)
1603 {
1604     gt_cval_write(env, ri, GTIMER_HYP, value);
1605 }
1606
1607 static uint64_t gt_hyp_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1608 {
1609     return gt_tval_read(env, ri, GTIMER_HYP);
1610 }
1611
1612 static void gt_hyp_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1613                               uint64_t value)
1614 {
1615     gt_tval_write(env, ri, GTIMER_HYP, value);
1616 }
1617
1618 static void gt_hyp_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1619                               uint64_t value)
1620 {
1621     gt_ctl_write(env, ri, GTIMER_HYP, value);
1622 }
1623
1624 static void gt_sec_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1625 {
1626     gt_timer_reset(env, ri, GTIMER_SEC);
1627 }
1628
1629 static void gt_sec_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1630                               uint64_t value)
1631 {
1632     gt_cval_write(env, ri, GTIMER_SEC, value);
1633 }
1634
1635 static uint64_t gt_sec_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1636 {
1637     return gt_tval_read(env, ri, GTIMER_SEC);
1638 }
1639
1640 static void gt_sec_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1641                               uint64_t value)
1642 {
1643     gt_tval_write(env, ri, GTIMER_SEC, value);
1644 }
1645
1646 static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1647                               uint64_t value)
1648 {
1649     gt_ctl_write(env, ri, GTIMER_SEC, value);
1650 }
1651
1652 void arm_gt_ptimer_cb(void *opaque)
1653 {
1654     ARMCPU *cpu = opaque;
1655
1656     gt_recalc_timer(cpu, GTIMER_PHYS);
1657 }
1658
1659 void arm_gt_vtimer_cb(void *opaque)
1660 {
1661     ARMCPU *cpu = opaque;
1662
1663     gt_recalc_timer(cpu, GTIMER_VIRT);
1664 }
1665
1666 void arm_gt_htimer_cb(void *opaque)
1667 {
1668     ARMCPU *cpu = opaque;
1669
1670     gt_recalc_timer(cpu, GTIMER_HYP);
1671 }
1672
1673 void arm_gt_stimer_cb(void *opaque)
1674 {
1675     ARMCPU *cpu = opaque;
1676
1677     gt_recalc_timer(cpu, GTIMER_SEC);
1678 }
1679
1680 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1681     /* Note that CNTFRQ is purely reads-as-written for the benefit
1682      * of software; writing it doesn't actually change the timer frequency.
1683      * Our reset value matches the fixed frequency we implement the timer at.
1684      */
1685     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
1686       .type = ARM_CP_ALIAS,
1687       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1688       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
1689     },
1690     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
1691       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
1692       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1693       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
1694       .resetvalue = (1000 * 1000 * 1000) / GTIMER_SCALE,
1695     },
1696     /* overall control: mostly access permissions */
1697     { .name = "CNTKCTL", .state = ARM_CP_STATE_BOTH,
1698       .opc0 = 3, .opc1 = 0, .crn = 14, .crm = 1, .opc2 = 0,
1699       .access = PL1_RW,
1700       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntkctl),
1701       .resetvalue = 0,
1702     },
1703     /* per-timer control */
1704     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
1705       .secure = ARM_CP_SECSTATE_NS,
1706       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1707       .accessfn = gt_ptimer_access,
1708       .fieldoffset = offsetoflow32(CPUARMState,
1709                                    cp15.c14_timer[GTIMER_PHYS].ctl),
1710       .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
1711     },
1712     { .name = "CNTP_CTL(S)",
1713       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
1714       .secure = ARM_CP_SECSTATE_S,
1715       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1716       .accessfn = gt_ptimer_access,
1717       .fieldoffset = offsetoflow32(CPUARMState,
1718                                    cp15.c14_timer[GTIMER_SEC].ctl),
1719       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
1720     },
1721     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
1722       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
1723       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1724       .accessfn = gt_ptimer_access,
1725       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
1726       .resetvalue = 0,
1727       .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
1728     },
1729     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
1730       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1731       .accessfn = gt_vtimer_access,
1732       .fieldoffset = offsetoflow32(CPUARMState,
1733                                    cp15.c14_timer[GTIMER_VIRT].ctl),
1734       .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
1735     },
1736     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
1737       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
1738       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1739       .accessfn = gt_vtimer_access,
1740       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
1741       .resetvalue = 0,
1742       .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
1743     },
1744     /* TimerValue views: a 32 bit downcounting view of the underlying state */
1745     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
1746       .secure = ARM_CP_SECSTATE_NS,
1747       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1748       .accessfn = gt_ptimer_access,
1749       .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
1750     },
1751     { .name = "CNTP_TVAL(S)",
1752       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
1753       .secure = ARM_CP_SECSTATE_S,
1754       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1755       .accessfn = gt_ptimer_access,
1756       .readfn = gt_sec_tval_read, .writefn = gt_sec_tval_write,
1757     },
1758     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1759       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
1760       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1761       .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset,
1762       .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
1763     },
1764     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
1765       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1766       .accessfn = gt_vtimer_access,
1767       .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
1768     },
1769     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1770       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
1771       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1772       .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset,
1773       .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
1774     },
1775     /* The counter itself */
1776     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
1777       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1778       .accessfn = gt_pct_access,
1779       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1780     },
1781     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
1782       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
1783       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1784       .accessfn = gt_pct_access, .readfn = gt_cnt_read,
1785     },
1786     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
1787       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1788       .accessfn = gt_vct_access,
1789       .readfn = gt_virt_cnt_read, .resetfn = arm_cp_reset_ignore,
1790     },
1791     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
1792       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
1793       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1794       .accessfn = gt_vct_access, .readfn = gt_virt_cnt_read,
1795     },
1796     /* Comparison value, indicating when the timer goes off */
1797     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
1798       .secure = ARM_CP_SECSTATE_NS,
1799       .access = PL1_RW | PL0_R,
1800       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1801       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1802       .accessfn = gt_ptimer_access,
1803       .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
1804     },
1805     { .name = "CNTP_CVAL(S)", .cp = 15, .crm = 14, .opc1 = 2,
1806       .secure = ARM_CP_SECSTATE_S,
1807       .access = PL1_RW | PL0_R,
1808       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1809       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
1810       .accessfn = gt_ptimer_access,
1811       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
1812     },
1813     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1814       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
1815       .access = PL1_RW | PL0_R,
1816       .type = ARM_CP_IO,
1817       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1818       .resetvalue = 0, .accessfn = gt_ptimer_access,
1819       .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
1820     },
1821     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
1822       .access = PL1_RW | PL0_R,
1823       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1824       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1825       .accessfn = gt_vtimer_access,
1826       .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
1827     },
1828     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1829       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
1830       .access = PL1_RW | PL0_R,
1831       .type = ARM_CP_IO,
1832       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1833       .resetvalue = 0, .accessfn = gt_vtimer_access,
1834       .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
1835     },
1836     /* Secure timer -- this is actually restricted to only EL3
1837      * and configurably Secure-EL1 via the accessfn.
1838      */
1839     { .name = "CNTPS_TVAL_EL1", .state = ARM_CP_STATE_AA64,
1840       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 0,
1841       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW,
1842       .accessfn = gt_stimer_access,
1843       .readfn = gt_sec_tval_read,
1844       .writefn = gt_sec_tval_write,
1845       .resetfn = gt_sec_timer_reset,
1846     },
1847     { .name = "CNTPS_CTL_EL1", .state = ARM_CP_STATE_AA64,
1848       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 1,
1849       .type = ARM_CP_IO, .access = PL1_RW,
1850       .accessfn = gt_stimer_access,
1851       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].ctl),
1852       .resetvalue = 0,
1853       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
1854     },
1855     { .name = "CNTPS_CVAL_EL1", .state = ARM_CP_STATE_AA64,
1856       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 2,
1857       .type = ARM_CP_IO, .access = PL1_RW,
1858       .accessfn = gt_stimer_access,
1859       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
1860       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
1861     },
1862     REGINFO_SENTINEL
1863 };
1864
1865 #else
1866 /* In user-mode none of the generic timer registers are accessible,
1867  * and their implementation depends on QEMU_CLOCK_VIRTUAL and qdev gpio outputs,
1868  * so instead just don't register any of them.
1869  */
1870 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1871     REGINFO_SENTINEL
1872 };
1873
1874 #endif
1875
1876 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1877 {
1878     if (arm_feature(env, ARM_FEATURE_LPAE)) {
1879         raw_write(env, ri, value);
1880     } else if (arm_feature(env, ARM_FEATURE_V7)) {
1881         raw_write(env, ri, value & 0xfffff6ff);
1882     } else {
1883         raw_write(env, ri, value & 0xfffff1ff);
1884     }
1885 }
1886
1887 #ifndef CONFIG_USER_ONLY
1888 /* get_phys_addr() isn't present for user-mode-only targets */
1889
1890 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri,
1891                                  bool isread)
1892 {
1893     if (ri->opc2 & 4) {
1894         /* The ATS12NSO* operations must trap to EL3 if executed in
1895          * Secure EL1 (which can only happen if EL3 is AArch64).
1896          * They are simply UNDEF if executed from NS EL1.
1897          * They function normally from EL2 or EL3.
1898          */
1899         if (arm_current_el(env) == 1) {
1900             if (arm_is_secure_below_el3(env)) {
1901                 return CP_ACCESS_TRAP_UNCATEGORIZED_EL3;
1902             }
1903             return CP_ACCESS_TRAP_UNCATEGORIZED;
1904         }
1905     }
1906     return CP_ACCESS_OK;
1907 }
1908
1909 static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
1910                              int access_type, ARMMMUIdx mmu_idx)
1911 {
1912     hwaddr phys_addr;
1913     target_ulong page_size;
1914     int prot;
1915     uint32_t fsr;
1916     bool ret;
1917     uint64_t par64;
1918     MemTxAttrs attrs = {};
1919     ARMMMUFaultInfo fi = {};
1920
1921     ret = get_phys_addr(env, value, access_type, mmu_idx,
1922                         &phys_addr, &attrs, &prot, &page_size, &fsr, &fi);
1923     if (extended_addresses_enabled(env)) {
1924         /* fsr is a DFSR/IFSR value for the long descriptor
1925          * translation table format, but with WnR always clear.
1926          * Convert it to a 64-bit PAR.
1927          */
1928         par64 = (1 << 11); /* LPAE bit always set */
1929         if (!ret) {
1930             par64 |= phys_addr & ~0xfffULL;
1931             if (!attrs.secure) {
1932                 par64 |= (1 << 9); /* NS */
1933             }
1934             /* We don't set the ATTR or SH fields in the PAR. */
1935         } else {
1936             par64 |= 1; /* F */
1937             par64 |= (fsr & 0x3f) << 1; /* FS */
1938             /* Note that S2WLK and FSTAGE are always zero, because we don't
1939              * implement virtualization and therefore there can't be a stage 2
1940              * fault.
1941              */
1942         }
1943     } else {
1944         /* fsr is a DFSR/IFSR value for the short descriptor
1945          * translation table format (with WnR always clear).
1946          * Convert it to a 32-bit PAR.
1947          */
1948         if (!ret) {
1949             /* We do not set any attribute bits in the PAR */
1950             if (page_size == (1 << 24)
1951                 && arm_feature(env, ARM_FEATURE_V7)) {
1952                 par64 = (phys_addr & 0xff000000) | (1 << 1);
1953             } else {
1954                 par64 = phys_addr & 0xfffff000;
1955             }
1956             if (!attrs.secure) {
1957                 par64 |= (1 << 9); /* NS */
1958             }
1959         } else {
1960             par64 = ((fsr & (1 << 10)) >> 5) | ((fsr & (1 << 12)) >> 6) |
1961                     ((fsr & 0xf) << 1) | 1;
1962         }
1963     }
1964     return par64;
1965 }
1966
1967 static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1968 {
1969     int access_type = ri->opc2 & 1;
1970     uint64_t par64;
1971     ARMMMUIdx mmu_idx;
1972     int el = arm_current_el(env);
1973     bool secure = arm_is_secure_below_el3(env);
1974
1975     switch (ri->opc2 & 6) {
1976     case 0:
1977         /* stage 1 current state PL1: ATS1CPR, ATS1CPW */
1978         switch (el) {
1979         case 3:
1980             mmu_idx = ARMMMUIdx_S1E3;
1981             break;
1982         case 2:
1983             mmu_idx = ARMMMUIdx_S1NSE1;
1984             break;
1985         case 1:
1986             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
1987             break;
1988         default:
1989             g_assert_not_reached();
1990         }
1991         break;
1992     case 2:
1993         /* stage 1 current state PL0: ATS1CUR, ATS1CUW */
1994         switch (el) {
1995         case 3:
1996             mmu_idx = ARMMMUIdx_S1SE0;
1997             break;
1998         case 2:
1999             mmu_idx = ARMMMUIdx_S1NSE0;
2000             break;
2001         case 1:
2002             mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
2003             break;
2004         default:
2005             g_assert_not_reached();
2006         }
2007         break;
2008     case 4:
2009         /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */
2010         mmu_idx = ARMMMUIdx_S12NSE1;
2011         break;
2012     case 6:
2013         /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */
2014         mmu_idx = ARMMMUIdx_S12NSE0;
2015         break;
2016     default:
2017         g_assert_not_reached();
2018     }
2019
2020     par64 = do_ats_write(env, value, access_type, mmu_idx);
2021
2022     A32_BANKED_CURRENT_REG_SET(env, par, par64);
2023 }
2024
2025 static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri,
2026                         uint64_t value)
2027 {
2028     int access_type = ri->opc2 & 1;
2029     uint64_t par64;
2030
2031     par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S2NS);
2032
2033     A32_BANKED_CURRENT_REG_SET(env, par, par64);
2034 }
2035
2036 static CPAccessResult at_s1e2_access(CPUARMState *env, const ARMCPRegInfo *ri,
2037                                      bool isread)
2038 {
2039     if (arm_current_el(env) == 3 && !(env->cp15.scr_el3 & SCR_NS)) {
2040         return CP_ACCESS_TRAP;
2041     }
2042     return CP_ACCESS_OK;
2043 }
2044
2045 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
2046                         uint64_t value)
2047 {
2048     int access_type = ri->opc2 & 1;
2049     ARMMMUIdx mmu_idx;
2050     int secure = arm_is_secure_below_el3(env);
2051
2052     switch (ri->opc2 & 6) {
2053     case 0:
2054         switch (ri->opc1) {
2055         case 0: /* AT S1E1R, AT S1E1W */
2056             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
2057             break;
2058         case 4: /* AT S1E2R, AT S1E2W */
2059             mmu_idx = ARMMMUIdx_S1E2;
2060             break;
2061         case 6: /* AT S1E3R, AT S1E3W */
2062             mmu_idx = ARMMMUIdx_S1E3;
2063             break;
2064         default:
2065             g_assert_not_reached();
2066         }
2067         break;
2068     case 2: /* AT S1E0R, AT S1E0W */
2069         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
2070         break;
2071     case 4: /* AT S12E1R, AT S12E1W */
2072         mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1;
2073         break;
2074     case 6: /* AT S12E0R, AT S12E0W */
2075         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0;
2076         break;
2077     default:
2078         g_assert_not_reached();
2079     }
2080
2081     env->cp15.par_el[1] = do_ats_write(env, value, access_type, mmu_idx);
2082 }
2083 #endif
2084
2085 static const ARMCPRegInfo vapa_cp_reginfo[] = {
2086     { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
2087       .access = PL1_RW, .resetvalue = 0,
2088       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
2089                              offsetoflow32(CPUARMState, cp15.par_ns) },
2090       .writefn = par_write },
2091 #ifndef CONFIG_USER_ONLY
2092     /* This underdecoding is safe because the reginfo is NO_RAW. */
2093     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
2094       .access = PL1_W, .accessfn = ats_access,
2095       .writefn = ats_write, .type = ARM_CP_NO_RAW },
2096 #endif
2097     REGINFO_SENTINEL
2098 };
2099
2100 /* Return basic MPU access permission bits.  */
2101 static uint32_t simple_mpu_ap_bits(uint32_t val)
2102 {
2103     uint32_t ret;
2104     uint32_t mask;
2105     int i;
2106     ret = 0;
2107     mask = 3;
2108     for (i = 0; i < 16; i += 2) {
2109         ret |= (val >> i) & mask;
2110         mask <<= 2;
2111     }
2112     return ret;
2113 }
2114
2115 /* Pad basic MPU access permission bits to extended format.  */
2116 static uint32_t extended_mpu_ap_bits(uint32_t val)
2117 {
2118     uint32_t ret;
2119     uint32_t mask;
2120     int i;
2121     ret = 0;
2122     mask = 3;
2123     for (i = 0; i < 16; i += 2) {
2124         ret |= (val & mask) << i;
2125         mask <<= 2;
2126     }
2127     return ret;
2128 }
2129
2130 static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
2131                                  uint64_t value)
2132 {
2133     env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
2134 }
2135
2136 static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
2137 {
2138     return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
2139 }
2140
2141 static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
2142                                  uint64_t value)
2143 {
2144     env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
2145 }
2146
2147 static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
2148 {
2149     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
2150 }
2151
2152 static uint64_t pmsav7_read(CPUARMState *env, const ARMCPRegInfo *ri)
2153 {
2154     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
2155
2156     if (!u32p) {
2157         return 0;
2158     }
2159
2160     u32p += env->cp15.c6_rgnr;
2161     return *u32p;
2162 }
2163
2164 static void pmsav7_write(CPUARMState *env, const ARMCPRegInfo *ri,
2165                          uint64_t value)
2166 {
2167     ARMCPU *cpu = arm_env_get_cpu(env);
2168     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
2169
2170     if (!u32p) {
2171         return;
2172     }
2173
2174     u32p += env->cp15.c6_rgnr;
2175     tlb_flush(CPU(cpu), 1); /* Mappings may have changed - purge! */
2176     *u32p = value;
2177 }
2178
2179 static void pmsav7_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2180 {
2181     ARMCPU *cpu = arm_env_get_cpu(env);
2182     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
2183
2184     if (!u32p) {
2185         return;
2186     }
2187
2188     memset(u32p, 0, sizeof(*u32p) * cpu->pmsav7_dregion);
2189 }
2190
2191 static void pmsav7_rgnr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2192                               uint64_t value)
2193 {
2194     ARMCPU *cpu = arm_env_get_cpu(env);
2195     uint32_t nrgs = cpu->pmsav7_dregion;
2196
2197     if (value >= nrgs) {
2198         qemu_log_mask(LOG_GUEST_ERROR,
2199                       "PMSAv7 RGNR write >= # supported regions, %" PRIu32
2200                       " > %" PRIu32 "\n", (uint32_t)value, nrgs);
2201         return;
2202     }
2203
2204     raw_write(env, ri, value);
2205 }
2206
2207 static const ARMCPRegInfo pmsav7_cp_reginfo[] = {
2208     { .name = "DRBAR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 0,
2209       .access = PL1_RW, .type = ARM_CP_NO_RAW,
2210       .fieldoffset = offsetof(CPUARMState, pmsav7.drbar),
2211       .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
2212     { .name = "DRSR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 2,
2213       .access = PL1_RW, .type = ARM_CP_NO_RAW,
2214       .fieldoffset = offsetof(CPUARMState, pmsav7.drsr),
2215       .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
2216     { .name = "DRACR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 4,
2217       .access = PL1_RW, .type = ARM_CP_NO_RAW,
2218       .fieldoffset = offsetof(CPUARMState, pmsav7.dracr),
2219       .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
2220     { .name = "RGNR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 2, .opc2 = 0,
2221       .access = PL1_RW,
2222       .fieldoffset = offsetof(CPUARMState, cp15.c6_rgnr),
2223       .writefn = pmsav7_rgnr_write },
2224     REGINFO_SENTINEL
2225 };
2226
2227 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
2228     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
2229       .access = PL1_RW, .type = ARM_CP_ALIAS,
2230       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
2231       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
2232     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
2233       .access = PL1_RW, .type = ARM_CP_ALIAS,
2234       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
2235       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
2236     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
2237       .access = PL1_RW,
2238       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
2239       .resetvalue = 0, },
2240     { .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
2241       .access = PL1_RW,
2242       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
2243       .resetvalue = 0, },
2244     { .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
2245       .access = PL1_RW,
2246       .fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
2247     { .name = "ICACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
2248       .access = PL1_RW,
2249       .fieldoffset = offsetof(CPUARMState, cp15.c2_insn), .resetvalue = 0, },
2250     /* Protection region base and size registers */
2251     { .name = "946_PRBS0", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0,
2252       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2253       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[0]) },
2254     { .name = "946_PRBS1", .cp = 15, .crn = 6, .crm = 1, .opc1 = 0,
2255       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2256       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[1]) },
2257     { .name = "946_PRBS2", .cp = 15, .crn = 6, .crm = 2, .opc1 = 0,
2258       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2259       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[2]) },
2260     { .name = "946_PRBS3", .cp = 15, .crn = 6, .crm = 3, .opc1 = 0,
2261       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2262       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[3]) },
2263     { .name = "946_PRBS4", .cp = 15, .crn = 6, .crm = 4, .opc1 = 0,
2264       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2265       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[4]) },
2266     { .name = "946_PRBS5", .cp = 15, .crn = 6, .crm = 5, .opc1 = 0,
2267       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2268       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[5]) },
2269     { .name = "946_PRBS6", .cp = 15, .crn = 6, .crm = 6, .opc1 = 0,
2270       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2271       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[6]) },
2272     { .name = "946_PRBS7", .cp = 15, .crn = 6, .crm = 7, .opc1 = 0,
2273       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2274       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[7]) },
2275     REGINFO_SENTINEL
2276 };
2277
2278 static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
2279                                  uint64_t value)
2280 {
2281     TCR *tcr = raw_ptr(env, ri);
2282     int maskshift = extract32(value, 0, 3);
2283
2284     if (!arm_feature(env, ARM_FEATURE_V8)) {
2285         if (arm_feature(env, ARM_FEATURE_LPAE) && (value & TTBCR_EAE)) {
2286             /* Pre ARMv8 bits [21:19], [15:14] and [6:3] are UNK/SBZP when
2287              * using Long-desciptor translation table format */
2288             value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
2289         } else if (arm_feature(env, ARM_FEATURE_EL3)) {
2290             /* In an implementation that includes the Security Extensions
2291              * TTBCR has additional fields PD0 [4] and PD1 [5] for
2292              * Short-descriptor translation table format.
2293              */
2294             value &= TTBCR_PD1 | TTBCR_PD0 | TTBCR_N;
2295         } else {
2296             value &= TTBCR_N;
2297         }
2298     }
2299
2300     /* Update the masks corresponding to the TCR bank being written
2301      * Note that we always calculate mask and base_mask, but
2302      * they are only used for short-descriptor tables (ie if EAE is 0);
2303      * for long-descriptor tables the TCR fields are used differently
2304      * and the mask and base_mask values are meaningless.
2305      */
2306     tcr->raw_tcr = value;
2307     tcr->mask = ~(((uint32_t)0xffffffffu) >> maskshift);
2308     tcr->base_mask = ~((uint32_t)0x3fffu >> maskshift);
2309 }
2310
2311 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2312                              uint64_t value)
2313 {
2314     ARMCPU *cpu = arm_env_get_cpu(env);
2315
2316     if (arm_feature(env, ARM_FEATURE_LPAE)) {
2317         /* With LPAE the TTBCR could result in a change of ASID
2318          * via the TTBCR.A1 bit, so do a TLB flush.
2319          */
2320         tlb_flush(CPU(cpu), 1);
2321     }
2322     vmsa_ttbcr_raw_write(env, ri, value);
2323 }
2324
2325 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2326 {
2327     TCR *tcr = raw_ptr(env, ri);
2328
2329     /* Reset both the TCR as well as the masks corresponding to the bank of
2330      * the TCR being reset.
2331      */
2332     tcr->raw_tcr = 0;
2333     tcr->mask = 0;
2334     tcr->base_mask = 0xffffc000u;
2335 }
2336
2337 static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2338                                uint64_t value)
2339 {
2340     ARMCPU *cpu = arm_env_get_cpu(env);
2341     TCR *tcr = raw_ptr(env, ri);
2342
2343     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
2344     tlb_flush(CPU(cpu), 1);
2345     tcr->raw_tcr = value;
2346 }
2347
2348 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2349                             uint64_t value)
2350 {
2351     /* 64 bit accesses to the TTBRs can change the ASID and so we
2352      * must flush the TLB.
2353      */
2354     if (cpreg_field_is_64bit(ri)) {
2355         ARMCPU *cpu = arm_env_get_cpu(env);
2356
2357         tlb_flush(CPU(cpu), 1);
2358     }
2359     raw_write(env, ri, value);
2360 }
2361
2362 static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2363                         uint64_t value)
2364 {
2365     ARMCPU *cpu = arm_env_get_cpu(env);
2366     CPUState *cs = CPU(cpu);
2367
2368     /* Accesses to VTTBR may change the VMID so we must flush the TLB.  */
2369     if (raw_read(env, ri) != value) {
2370         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
2371                             ARMMMUIdx_S2NS, -1);
2372         raw_write(env, ri, value);
2373     }
2374 }
2375
2376 static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
2377     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
2378       .access = PL1_RW, .type = ARM_CP_ALIAS,
2379       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
2380                              offsetoflow32(CPUARMState, cp15.dfsr_ns) }, },
2381     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
2382       .access = PL1_RW, .resetvalue = 0,
2383       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
2384                              offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
2385     { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
2386       .access = PL1_RW, .resetvalue = 0,
2387       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
2388                              offsetof(CPUARMState, cp15.dfar_ns) } },
2389     { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
2390       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
2391       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
2392       .resetvalue = 0, },
2393     REGINFO_SENTINEL
2394 };
2395
2396 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
2397     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
2398       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
2399       .access = PL1_RW,
2400       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
2401     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
2402       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
2403       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
2404       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
2405                              offsetof(CPUARMState, cp15.ttbr0_ns) } },
2406     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
2407       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
2408       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
2409       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
2410                              offsetof(CPUARMState, cp15.ttbr1_ns) } },
2411     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
2412       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
2413       .access = PL1_RW, .writefn = vmsa_tcr_el1_write,
2414       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
2415       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
2416     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
2417       .access = PL1_RW, .type = ARM_CP_ALIAS, .writefn = vmsa_ttbcr_write,
2418       .raw_writefn = vmsa_ttbcr_raw_write,
2419       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
2420                              offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
2421     REGINFO_SENTINEL
2422 };
2423
2424 static void omap_ticonfig_write(CPUARMState *env, const ARMCPRegInfo *ri,
2425                                 uint64_t value)
2426 {
2427     env->cp15.c15_ticonfig = value & 0xe7;
2428     /* The OS_TYPE bit in this register changes the reported CPUID! */
2429     env->cp15.c0_cpuid = (value & (1 << 5)) ?
2430         ARM_CPUID_TI915T : ARM_CPUID_TI925T;
2431 }
2432
2433 static void omap_threadid_write(CPUARMState *env, const ARMCPRegInfo *ri,
2434                                 uint64_t value)
2435 {
2436     env->cp15.c15_threadid = value & 0xffff;
2437 }
2438
2439 static void omap_wfi_write(CPUARMState *env, const ARMCPRegInfo *ri,
2440                            uint64_t value)
2441 {
2442     /* Wait-for-interrupt (deprecated) */
2443     cpu_interrupt(CPU(arm_env_get_cpu(env)), CPU_INTERRUPT_HALT);
2444 }
2445
2446 static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
2447                                   uint64_t value)
2448 {
2449     /* On OMAP there are registers indicating the max/min index of dcache lines
2450      * containing a dirty line; cache flush operations have to reset these.
2451      */
2452     env->cp15.c15_i_max = 0x000;
2453     env->cp15.c15_i_min = 0xff0;
2454 }
2455
2456 static const ARMCPRegInfo omap_cp_reginfo[] = {
2457     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
2458       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
2459       .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
2460       .resetvalue = 0, },
2461     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
2462       .access = PL1_RW, .type = ARM_CP_NOP },
2463     { .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
2464       .access = PL1_RW,
2465       .fieldoffset = offsetof(CPUARMState, cp15.c15_ticonfig), .resetvalue = 0,
2466       .writefn = omap_ticonfig_write },
2467     { .name = "IMAX", .cp = 15, .crn = 15, .crm = 2, .opc1 = 0, .opc2 = 0,
2468       .access = PL1_RW,
2469       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_max), .resetvalue = 0, },
2470     { .name = "IMIN", .cp = 15, .crn = 15, .crm = 3, .opc1 = 0, .opc2 = 0,
2471       .access = PL1_RW, .resetvalue = 0xff0,
2472       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_min) },
2473     { .name = "THREADID", .cp = 15, .crn = 15, .crm = 4, .opc1 = 0, .opc2 = 0,
2474       .access = PL1_RW,
2475       .fieldoffset = offsetof(CPUARMState, cp15.c15_threadid), .resetvalue = 0,
2476       .writefn = omap_threadid_write },
2477     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
2478       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
2479       .type = ARM_CP_NO_RAW,
2480       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
2481     /* TODO: Peripheral port remap register:
2482      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
2483      * base address at $rn & ~0xfff and map size of 0x200 << ($rn & 0xfff),
2484      * when MMU is off.
2485      */
2486     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
2487       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
2488       .type = ARM_CP_OVERRIDE | ARM_CP_NO_RAW,
2489       .writefn = omap_cachemaint_write },
2490     { .name = "C9", .cp = 15, .crn = 9,
2491       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
2492       .type = ARM_CP_CONST | ARM_CP_OVERRIDE, .resetvalue = 0 },
2493     REGINFO_SENTINEL
2494 };
2495
2496 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
2497                               uint64_t value)
2498 {
2499     env->cp15.c15_cpar = value & 0x3fff;
2500 }
2501
2502 static const ARMCPRegInfo xscale_cp_reginfo[] = {
2503     { .name = "XSCALE_CPAR",
2504       .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
2505       .fieldoffset = offsetof(CPUARMState, cp15.c15_cpar), .resetvalue = 0,
2506       .writefn = xscale_cpar_write, },
2507     { .name = "XSCALE_AUXCR",
2508       .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW,
2509       .fieldoffset = offsetof(CPUARMState, cp15.c1_xscaleauxcr),
2510       .resetvalue = 0, },
2511     /* XScale specific cache-lockdown: since we have no cache we NOP these
2512      * and hope the guest does not really rely on cache behaviour.
2513      */
2514     { .name = "XSCALE_LOCK_ICACHE_LINE",
2515       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
2516       .access = PL1_W, .type = ARM_CP_NOP },
2517     { .name = "XSCALE_UNLOCK_ICACHE",
2518       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
2519       .access = PL1_W, .type = ARM_CP_NOP },
2520     { .name = "XSCALE_DCACHE_LOCK",
2521       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 0,
2522       .access = PL1_RW, .type = ARM_CP_NOP },
2523     { .name = "XSCALE_UNLOCK_DCACHE",
2524       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 1,
2525       .access = PL1_W, .type = ARM_CP_NOP },
2526     REGINFO_SENTINEL
2527 };
2528
2529 static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
2530     /* RAZ/WI the whole crn=15 space, when we don't have a more specific
2531      * implementation of this implementation-defined space.
2532      * Ideally this should eventually disappear in favour of actually
2533      * implementing the correct behaviour for all cores.
2534      */
2535     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
2536       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
2537       .access = PL1_RW,
2538       .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
2539       .resetvalue = 0 },
2540     REGINFO_SENTINEL
2541 };
2542
2543 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
2544     /* Cache status: RAZ because we have no cache so it's always clean */
2545     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
2546       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2547       .resetvalue = 0 },
2548     REGINFO_SENTINEL
2549 };
2550
2551 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
2552     /* We never have a a block transfer operation in progress */
2553     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
2554       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2555       .resetvalue = 0 },
2556     /* The cache ops themselves: these all NOP for QEMU */
2557     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
2558       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2559     { .name = "IDCR", .cp = 15, .crm = 6, .opc1 = 0,
2560       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2561     { .name = "CDCR", .cp = 15, .crm = 12, .opc1 = 0,
2562       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2563     { .name = "PIR", .cp = 15, .crm = 12, .opc1 = 1,
2564       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2565     { .name = "PDR", .cp = 15, .crm = 12, .opc1 = 2,
2566       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2567     { .name = "CIDCR", .cp = 15, .crm = 14, .opc1 = 0,
2568       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2569     REGINFO_SENTINEL
2570 };
2571
2572 static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
2573     /* The cache test-and-clean instructions always return (1 << 30)
2574      * to indicate that there are no dirty cache lines.
2575      */
2576     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
2577       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2578       .resetvalue = (1 << 30) },
2579     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
2580       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2581       .resetvalue = (1 << 30) },
2582     REGINFO_SENTINEL
2583 };
2584
2585 static const ARMCPRegInfo strongarm_cp_reginfo[] = {
2586     /* Ignore ReadBuffer accesses */
2587     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
2588       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
2589       .access = PL1_RW, .resetvalue = 0,
2590       .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_RAW },
2591     REGINFO_SENTINEL
2592 };
2593
2594 static uint64_t midr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2595 {
2596     ARMCPU *cpu = arm_env_get_cpu(env);
2597     unsigned int cur_el = arm_current_el(env);
2598     bool secure = arm_is_secure(env);
2599
2600     if (arm_feature(&cpu->env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
2601         return env->cp15.vpidr_el2;
2602     }
2603     return raw_read(env, ri);
2604 }
2605
2606 static uint64_t mpidr_read_val(CPUARMState *env)
2607 {
2608     ARMCPU *cpu = ARM_CPU(arm_env_get_cpu(env));
2609     uint64_t mpidr = cpu->mp_affinity;
2610
2611     if (arm_feature(env, ARM_FEATURE_V7MP)) {
2612         mpidr |= (1U << 31);
2613         /* Cores which are uniprocessor (non-coherent)
2614          * but still implement the MP extensions set
2615          * bit 30. (For instance, Cortex-R5).
2616          */
2617         if (cpu->mp_is_up) {
2618             mpidr |= (1u << 30);
2619         }
2620     }
2621     return mpidr;
2622 }
2623
2624 static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2625 {
2626     unsigned int cur_el = arm_current_el(env);
2627     bool secure = arm_is_secure(env);
2628
2629     if (arm_feature(env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
2630         return env->cp15.vmpidr_el2;
2631     }
2632     return mpidr_read_val(env);
2633 }
2634
2635 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
2636     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
2637       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
2638       .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_RAW },
2639     REGINFO_SENTINEL
2640 };
2641
2642 static const ARMCPRegInfo lpae_cp_reginfo[] = {
2643     /* NOP AMAIR0/1 */
2644     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
2645       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
2646       .access = PL1_RW, .type = ARM_CP_CONST,
2647       .resetvalue = 0 },
2648     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
2649     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
2650       .access = PL1_RW, .type = ARM_CP_CONST,
2651       .resetvalue = 0 },
2652     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
2653       .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
2654       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
2655                              offsetof(CPUARMState, cp15.par_ns)} },
2656     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
2657       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2658       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
2659                              offsetof(CPUARMState, cp15.ttbr0_ns) },
2660       .writefn = vmsa_ttbr_write, },
2661     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
2662       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2663       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
2664                              offsetof(CPUARMState, cp15.ttbr1_ns) },
2665       .writefn = vmsa_ttbr_write, },
2666     REGINFO_SENTINEL
2667 };
2668
2669 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2670 {
2671     return vfp_get_fpcr(env);
2672 }
2673
2674 static void aa64_fpcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2675                             uint64_t value)
2676 {
2677     vfp_set_fpcr(env, value);
2678 }
2679
2680 static uint64_t aa64_fpsr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2681 {
2682     return vfp_get_fpsr(env);
2683 }
2684
2685 static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2686                             uint64_t value)
2687 {
2688     vfp_set_fpsr(env, value);
2689 }
2690
2691 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri,
2692                                        bool isread)
2693 {
2694     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
2695         return CP_ACCESS_TRAP;
2696     }
2697     return CP_ACCESS_OK;
2698 }
2699
2700 static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
2701                             uint64_t value)
2702 {
2703     env->daif = value & PSTATE_DAIF;
2704 }
2705
2706 static CPAccessResult aa64_cacheop_access(CPUARMState *env,
2707                                           const ARMCPRegInfo *ri,
2708                                           bool isread)
2709 {
2710     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
2711      * SCTLR_EL1.UCI is set.
2712      */
2713     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) {
2714         return CP_ACCESS_TRAP;
2715     }
2716     return CP_ACCESS_OK;
2717 }
2718
2719 /* See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
2720  * Page D4-1736 (DDI0487A.b)
2721  */
2722
2723 static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2724                                     uint64_t value)
2725 {
2726     ARMCPU *cpu = arm_env_get_cpu(env);
2727     CPUState *cs = CPU(cpu);
2728
2729     if (arm_is_secure_below_el3(env)) {
2730         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2731     } else {
2732         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
2733     }
2734 }
2735
2736 static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2737                                       uint64_t value)
2738 {
2739     bool sec = arm_is_secure_below_el3(env);
2740     CPUState *other_cs;
2741
2742     CPU_FOREACH(other_cs) {
2743         if (sec) {
2744             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2745         } else {
2746             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
2747                                 ARMMMUIdx_S12NSE0, -1);
2748         }
2749     }
2750 }
2751
2752 static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2753                                   uint64_t value)
2754 {
2755     /* Note that the 'ALL' scope must invalidate both stage 1 and
2756      * stage 2 translations, whereas most other scopes only invalidate
2757      * stage 1 translations.
2758      */
2759     ARMCPU *cpu = arm_env_get_cpu(env);
2760     CPUState *cs = CPU(cpu);
2761
2762     if (arm_is_secure_below_el3(env)) {
2763         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2764     } else {
2765         if (arm_feature(env, ARM_FEATURE_EL2)) {
2766             tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
2767                                 ARMMMUIdx_S2NS, -1);
2768         } else {
2769             tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
2770         }
2771     }
2772 }
2773
2774 static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri,
2775                                   uint64_t value)
2776 {
2777     ARMCPU *cpu = arm_env_get_cpu(env);
2778     CPUState *cs = CPU(cpu);
2779
2780     tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E2, -1);
2781 }
2782
2783 static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri,
2784                                   uint64_t value)
2785 {
2786     ARMCPU *cpu = arm_env_get_cpu(env);
2787     CPUState *cs = CPU(cpu);
2788
2789     tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E3, -1);
2790 }
2791
2792 static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2793                                     uint64_t value)
2794 {
2795     /* Note that the 'ALL' scope must invalidate both stage 1 and
2796      * stage 2 translations, whereas most other scopes only invalidate
2797      * stage 1 translations.
2798      */
2799     bool sec = arm_is_secure_below_el3(env);
2800     bool has_el2 = arm_feature(env, ARM_FEATURE_EL2);
2801     CPUState *other_cs;
2802
2803     CPU_FOREACH(other_cs) {
2804         if (sec) {
2805             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2806         } else if (has_el2) {
2807             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
2808                                 ARMMMUIdx_S12NSE0, ARMMMUIdx_S2NS, -1);
2809         } else {
2810             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
2811                                 ARMMMUIdx_S12NSE0, -1);
2812         }
2813     }
2814 }
2815
2816 static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2817                                     uint64_t value)
2818 {
2819     CPUState *other_cs;
2820
2821     CPU_FOREACH(other_cs) {
2822         tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E2, -1);
2823     }
2824 }
2825
2826 static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2827                                     uint64_t value)
2828 {
2829     CPUState *other_cs;
2830
2831     CPU_FOREACH(other_cs) {
2832         tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E3, -1);
2833     }
2834 }
2835
2836 static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2837                                  uint64_t value)
2838 {
2839     /* Invalidate by VA, EL1&0 (AArch64 version).
2840      * Currently handles all of VAE1, VAAE1, VAALE1 and VALE1,
2841      * since we don't support flush-for-specific-ASID-only or
2842      * flush-last-level-only.
2843      */
2844     ARMCPU *cpu = arm_env_get_cpu(env);
2845     CPUState *cs = CPU(cpu);
2846     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2847
2848     if (arm_is_secure_below_el3(env)) {
2849         tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1SE1,
2850                                  ARMMMUIdx_S1SE0, -1);
2851     } else {
2852         tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S12NSE1,
2853                                  ARMMMUIdx_S12NSE0, -1);
2854     }
2855 }
2856
2857 static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
2858                                  uint64_t value)
2859 {
2860     /* Invalidate by VA, EL2
2861      * Currently handles both VAE2 and VALE2, since we don't support
2862      * flush-last-level-only.
2863      */
2864     ARMCPU *cpu = arm_env_get_cpu(env);
2865     CPUState *cs = CPU(cpu);
2866     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2867
2868     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E2, -1);
2869 }
2870
2871 static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
2872                                  uint64_t value)
2873 {
2874     /* Invalidate by VA, EL3
2875      * Currently handles both VAE3 and VALE3, since we don't support
2876      * flush-last-level-only.
2877      */
2878     ARMCPU *cpu = arm_env_get_cpu(env);
2879     CPUState *cs = CPU(cpu);
2880     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2881
2882     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E3, -1);
2883 }
2884
2885 static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2886                                    uint64_t value)
2887 {
2888     bool sec = arm_is_secure_below_el3(env);
2889     CPUState *other_cs;
2890     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2891
2892     CPU_FOREACH(other_cs) {
2893         if (sec) {
2894             tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1SE1,
2895                                      ARMMMUIdx_S1SE0, -1);
2896         } else {
2897             tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S12NSE1,
2898                                      ARMMMUIdx_S12NSE0, -1);
2899         }
2900     }
2901 }
2902
2903 static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2904                                    uint64_t value)
2905 {
2906     CPUState *other_cs;
2907     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2908
2909     CPU_FOREACH(other_cs) {
2910         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E2, -1);
2911     }
2912 }
2913
2914 static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2915                                    uint64_t value)
2916 {
2917     CPUState *other_cs;
2918     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2919
2920     CPU_FOREACH(other_cs) {
2921         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E3, -1);
2922     }
2923 }
2924
2925 static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2926                                     uint64_t value)
2927 {
2928     /* Invalidate by IPA. This has to invalidate any structures that
2929      * contain only stage 2 translation information, but does not need
2930      * to apply to structures that contain combined stage 1 and stage 2
2931      * translation information.
2932      * This must NOP if EL2 isn't implemented or SCR_EL3.NS is zero.
2933      */
2934     ARMCPU *cpu = arm_env_get_cpu(env);
2935     CPUState *cs = CPU(cpu);
2936     uint64_t pageaddr;
2937
2938     if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
2939         return;
2940     }
2941
2942     pageaddr = sextract64(value << 12, 0, 48);
2943
2944     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S2NS, -1);
2945 }
2946
2947 static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2948                                       uint64_t value)
2949 {
2950     CPUState *other_cs;
2951     uint64_t pageaddr;
2952
2953     if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
2954         return;
2955     }
2956
2957     pageaddr = sextract64(value << 12, 0, 48);
2958
2959     CPU_FOREACH(other_cs) {
2960         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S2NS, -1);
2961     }
2962 }
2963
2964 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
2965                                       bool isread)
2966 {
2967     /* We don't implement EL2, so the only control on DC ZVA is the
2968      * bit in the SCTLR which can prohibit access for EL0.
2969      */
2970     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
2971         return CP_ACCESS_TRAP;
2972     }
2973     return CP_ACCESS_OK;
2974 }
2975
2976 static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
2977 {
2978     ARMCPU *cpu = arm_env_get_cpu(env);
2979     int dzp_bit = 1 << 4;
2980
2981     /* DZP indicates whether DC ZVA access is allowed */
2982     if (aa64_zva_access(env, NULL, false) == CP_ACCESS_OK) {
2983         dzp_bit = 0;
2984     }
2985     return cpu->dcz_blocksize | dzp_bit;
2986 }
2987
2988 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
2989                                     bool isread)
2990 {
2991     if (!(env->pstate & PSTATE_SP)) {
2992         /* Access to SP_EL0 is undefined if it's being used as
2993          * the stack pointer.
2994          */
2995         return CP_ACCESS_TRAP_UNCATEGORIZED;
2996     }
2997     return CP_ACCESS_OK;
2998 }
2999
3000 static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
3001 {
3002     return env->pstate & PSTATE_SP;
3003 }
3004
3005 static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
3006 {
3007     update_spsel(env, val);
3008 }
3009
3010 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3011                         uint64_t value)
3012 {
3013     ARMCPU *cpu = arm_env_get_cpu(env);
3014
3015     if (raw_read(env, ri) == value) {
3016         /* Skip the TLB flush if nothing actually changed; Linux likes
3017          * to do a lot of pointless SCTLR writes.
3018          */
3019         return;
3020     }
3021
3022     raw_write(env, ri, value);
3023     /* ??? Lots of these bits are not implemented.  */
3024     /* This may enable/disable the MMU, so do a TLB flush.  */
3025     tlb_flush(CPU(cpu), 1);
3026 }
3027
3028 static CPAccessResult fpexc32_access(CPUARMState *env, const ARMCPRegInfo *ri,
3029                                      bool isread)
3030 {
3031     if ((env->cp15.cptr_el[2] & CPTR_TFP) && arm_current_el(env) == 2) {
3032         return CP_ACCESS_TRAP_FP_EL2;
3033     }
3034     if (env->cp15.cptr_el[3] & CPTR_TFP) {
3035         return CP_ACCESS_TRAP_FP_EL3;
3036     }
3037     return CP_ACCESS_OK;
3038 }
3039
3040 static const ARMCPRegInfo v8_cp_reginfo[] = {
3041     /* Minimal set of EL0-visible registers. This will need to be expanded
3042      * significantly for system emulation of AArch64 CPUs.
3043      */
3044     { .name = "NZCV", .state = ARM_CP_STATE_AA64,
3045       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
3046       .access = PL0_RW, .type = ARM_CP_NZCV },
3047     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
3048       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
3049       .type = ARM_CP_NO_RAW,
3050       .access = PL0_RW, .accessfn = aa64_daif_access,
3051       .fieldoffset = offsetof(CPUARMState, daif),
3052       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
3053     { .name = "FPCR", .state = ARM_CP_STATE_AA64,
3054       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
3055       .access = PL0_RW, .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
3056     { .name = "FPSR", .state = ARM_CP_STATE_AA64,
3057       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
3058       .access = PL0_RW, .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
3059     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
3060       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
3061       .access = PL0_R, .type = ARM_CP_NO_RAW,
3062       .readfn = aa64_dczid_read },
3063     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
3064       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
3065       .access = PL0_W, .type = ARM_CP_DC_ZVA,
3066 #ifndef CONFIG_USER_ONLY
3067       /* Avoid overhead of an access check that always passes in user-mode */
3068       .accessfn = aa64_zva_access,
3069 #endif
3070     },
3071     { .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
3072       .opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
3073       .access = PL1_R, .type = ARM_CP_CURRENTEL },
3074     /* Cache ops: all NOPs since we don't emulate caches */
3075     { .name = "IC_IALLUIS", .state = ARM_CP_STATE_AA64,
3076       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
3077       .access = PL1_W, .type = ARM_CP_NOP },
3078     { .name = "IC_IALLU", .state = ARM_CP_STATE_AA64,
3079       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
3080       .access = PL1_W, .type = ARM_CP_NOP },
3081     { .name = "IC_IVAU", .state = ARM_CP_STATE_AA64,
3082       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 5, .opc2 = 1,
3083       .access = PL0_W, .type = ARM_CP_NOP,
3084       .accessfn = aa64_cacheop_access },
3085     { .name = "DC_IVAC", .state = ARM_CP_STATE_AA64,
3086       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
3087       .access = PL1_W, .type = ARM_CP_NOP },
3088     { .name = "DC_ISW", .state = ARM_CP_STATE_AA64,
3089       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
3090       .access = PL1_W, .type = ARM_CP_NOP },
3091     { .name = "DC_CVAC", .state = ARM_CP_STATE_AA64,
3092       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 1,
3093       .access = PL0_W, .type = ARM_CP_NOP,
3094       .accessfn = aa64_cacheop_access },
3095     { .name = "DC_CSW", .state = ARM_CP_STATE_AA64,
3096       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
3097       .access = PL1_W, .type = ARM_CP_NOP },
3098     { .name = "DC_CVAU", .state = ARM_CP_STATE_AA64,
3099       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 11, .opc2 = 1,
3100       .access = PL0_W, .type = ARM_CP_NOP,
3101       .accessfn = aa64_cacheop_access },
3102     { .name = "DC_CIVAC", .state = ARM_CP_STATE_AA64,
3103       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 1,
3104       .access = PL0_W, .type = ARM_CP_NOP,
3105       .accessfn = aa64_cacheop_access },
3106     { .name = "DC_CISW", .state = ARM_CP_STATE_AA64,
3107       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
3108       .access = PL1_W, .type = ARM_CP_NOP },
3109     /* TLBI operations */
3110     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
3111       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
3112       .access = PL1_W, .type = ARM_CP_NO_RAW,
3113       .writefn = tlbi_aa64_vmalle1is_write },
3114     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
3115       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
3116       .access = PL1_W, .type = ARM_CP_NO_RAW,
3117       .writefn = tlbi_aa64_vae1is_write },
3118     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
3119       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
3120       .access = PL1_W, .type = ARM_CP_NO_RAW,
3121       .writefn = tlbi_aa64_vmalle1is_write },
3122     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
3123       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
3124       .access = PL1_W, .type = ARM_CP_NO_RAW,
3125       .writefn = tlbi_aa64_vae1is_write },
3126     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
3127       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
3128       .access = PL1_W, .type = ARM_CP_NO_RAW,
3129       .writefn = tlbi_aa64_vae1is_write },
3130     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
3131       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
3132       .access = PL1_W, .type = ARM_CP_NO_RAW,
3133       .writefn = tlbi_aa64_vae1is_write },
3134     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
3135       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
3136       .access = PL1_W, .type = ARM_CP_NO_RAW,
3137       .writefn = tlbi_aa64_vmalle1_write },
3138     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
3139       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
3140       .access = PL1_W, .type = ARM_CP_NO_RAW,
3141       .writefn = tlbi_aa64_vae1_write },
3142     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
3143       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
3144       .access = PL1_W, .type = ARM_CP_NO_RAW,
3145       .writefn = tlbi_aa64_vmalle1_write },
3146     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
3147       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
3148       .access = PL1_W, .type = ARM_CP_NO_RAW,
3149       .writefn = tlbi_aa64_vae1_write },
3150     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
3151       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
3152       .access = PL1_W, .type = ARM_CP_NO_RAW,
3153       .writefn = tlbi_aa64_vae1_write },
3154     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
3155       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
3156       .access = PL1_W, .type = ARM_CP_NO_RAW,
3157       .writefn = tlbi_aa64_vae1_write },
3158     { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
3159       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
3160       .access = PL2_W, .type = ARM_CP_NO_RAW,
3161       .writefn = tlbi_aa64_ipas2e1is_write },
3162     { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
3163       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
3164       .access = PL2_W, .type = ARM_CP_NO_RAW,
3165       .writefn = tlbi_aa64_ipas2e1is_write },
3166     { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
3167       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
3168       .access = PL2_W, .type = ARM_CP_NO_RAW,
3169       .writefn = tlbi_aa64_alle1is_write },
3170     { .name = "TLBI_VMALLS12E1IS", .state = ARM_CP_STATE_AA64,
3171       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 6,
3172       .access = PL2_W, .type = ARM_CP_NO_RAW,
3173       .writefn = tlbi_aa64_alle1is_write },
3174     { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
3175       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
3176       .access = PL2_W, .type = ARM_CP_NO_RAW,
3177       .writefn = tlbi_aa64_ipas2e1_write },
3178     { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
3179       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
3180       .access = PL2_W, .type = ARM_CP_NO_RAW,
3181       .writefn = tlbi_aa64_ipas2e1_write },
3182     { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
3183       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
3184       .access = PL2_W, .type = ARM_CP_NO_RAW,
3185       .writefn = tlbi_aa64_alle1_write },
3186     { .name = "TLBI_VMALLS12E1", .state = ARM_CP_STATE_AA64,
3187       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 6,
3188       .access = PL2_W, .type = ARM_CP_NO_RAW,
3189       .writefn = tlbi_aa64_alle1is_write },
3190 #ifndef CONFIG_USER_ONLY
3191     /* 64 bit address translation operations */
3192     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
3193       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
3194       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3195     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
3196       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
3197       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3198     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
3199       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
3200       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3201     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
3202       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
3203       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3204     { .name = "AT_S12E1R", .state = ARM_CP_STATE_AA64,
3205       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 4,
3206       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3207     { .name = "AT_S12E1W", .state = ARM_CP_STATE_AA64,
3208       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 5,
3209       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3210     { .name = "AT_S12E0R", .state = ARM_CP_STATE_AA64,
3211       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 6,
3212       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3213     { .name = "AT_S12E0W", .state = ARM_CP_STATE_AA64,
3214       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 7,
3215       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3216     /* AT S1E2* are elsewhere as they UNDEF from EL3 if EL2 is not present */
3217     { .name = "AT_S1E3R", .state = ARM_CP_STATE_AA64,
3218       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 0,
3219       .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3220     { .name = "AT_S1E3W", .state = ARM_CP_STATE_AA64,
3221       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 1,
3222       .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3223     { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
3224       .type = ARM_CP_ALIAS,
3225       .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
3226       .access = PL1_RW, .resetvalue = 0,
3227       .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
3228       .writefn = par_write },
3229 #endif
3230     /* TLB invalidate last level of translation table walk */
3231     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
3232       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
3233     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
3234       .type = ARM_CP_NO_RAW, .access = PL1_W,
3235       .writefn = tlbimvaa_is_write },
3236     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
3237       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
3238     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
3239       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
3240     /* 32 bit cache operations */
3241     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
3242       .type = ARM_CP_NOP, .access = PL1_W },
3243     { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
3244       .type = ARM_CP_NOP, .access = PL1_W },
3245     { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
3246       .type = ARM_CP_NOP, .access = PL1_W },
3247     { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
3248       .type = ARM_CP_NOP, .access = PL1_W },
3249     { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
3250       .type = ARM_CP_NOP, .access = PL1_W },
3251     { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
3252       .type = ARM_CP_NOP, .access = PL1_W },
3253     { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
3254       .type = ARM_CP_NOP, .access = PL1_W },
3255     { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
3256       .type = ARM_CP_NOP, .access = PL1_W },
3257     { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
3258       .type = ARM_CP_NOP, .access = PL1_W },
3259     { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
3260       .type = ARM_CP_NOP, .access = PL1_W },
3261     { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
3262       .type = ARM_CP_NOP, .access = PL1_W },
3263     { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
3264       .type = ARM_CP_NOP, .access = PL1_W },
3265     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
3266       .type = ARM_CP_NOP, .access = PL1_W },
3267     /* MMU Domain access control / MPU write buffer control */
3268     { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
3269       .access = PL1_RW, .resetvalue = 0,
3270       .writefn = dacr_write, .raw_writefn = raw_write,
3271       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
3272                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
3273     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
3274       .type = ARM_CP_ALIAS,
3275       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
3276       .access = PL1_RW,
3277       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
3278     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
3279       .type = ARM_CP_ALIAS,
3280       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
3281       .access = PL1_RW,
3282       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_SVC]) },
3283     /* We rely on the access checks not allowing the guest to write to the
3284      * state field when SPSel indicates that it's being used as the stack
3285      * pointer.
3286      */
3287     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
3288       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
3289       .access = PL1_RW, .accessfn = sp_el0_access,
3290       .type = ARM_CP_ALIAS,
3291       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
3292     { .name = "SP_EL1", .state = ARM_CP_STATE_AA64,
3293       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 1, .opc2 = 0,
3294       .access = PL2_RW, .type = ARM_CP_ALIAS,
3295       .fieldoffset = offsetof(CPUARMState, sp_el[1]) },
3296     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
3297       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
3298       .type = ARM_CP_NO_RAW,
3299       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
3300     { .name = "FPEXC32_EL2", .state = ARM_CP_STATE_AA64,
3301       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 3, .opc2 = 0,
3302       .type = ARM_CP_ALIAS,
3303       .fieldoffset = offsetof(CPUARMState, vfp.xregs[ARM_VFP_FPEXC]),
3304       .access = PL2_RW, .accessfn = fpexc32_access },
3305     { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
3306       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
3307       .access = PL2_RW, .resetvalue = 0,
3308       .writefn = dacr_write, .raw_writefn = raw_write,
3309       .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
3310     { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
3311       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
3312       .access = PL2_RW, .resetvalue = 0,
3313       .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
3314     { .name = "SPSR_IRQ", .state = ARM_CP_STATE_AA64,
3315       .type = ARM_CP_ALIAS,
3316       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 0,
3317       .access = PL2_RW,
3318       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_IRQ]) },
3319     { .name = "SPSR_ABT", .state = ARM_CP_STATE_AA64,
3320       .type = ARM_CP_ALIAS,
3321       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 1,
3322       .access = PL2_RW,
3323       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_ABT]) },
3324     { .name = "SPSR_UND", .state = ARM_CP_STATE_AA64,
3325       .type = ARM_CP_ALIAS,
3326       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 2,
3327       .access = PL2_RW,
3328       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_UND]) },
3329     { .name = "SPSR_FIQ", .state = ARM_CP_STATE_AA64,
3330       .type = ARM_CP_ALIAS,
3331       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 3,
3332       .access = PL2_RW,
3333       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_FIQ]) },
3334     REGINFO_SENTINEL
3335 };
3336
3337 /* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
3338 static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
3339     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
3340       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
3341       .access = PL2_RW,
3342       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
3343     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
3344       .type = ARM_CP_NO_RAW,
3345       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
3346       .access = PL2_RW,
3347       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
3348     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
3349       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
3350       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3351     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
3352       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
3353       .access = PL2_RW, .type = ARM_CP_CONST,
3354       .resetvalue = 0 },
3355     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3356       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
3357       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3358     { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
3359       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
3360       .access = PL2_RW, .type = ARM_CP_CONST,
3361       .resetvalue = 0 },
3362     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3363       .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
3364       .access = PL2_RW, .type = ARM_CP_CONST,
3365       .resetvalue = 0 },
3366     { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
3367       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
3368       .access = PL2_RW, .type = ARM_CP_CONST,
3369       .resetvalue = 0 },
3370     { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
3371       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
3372       .access = PL2_RW, .type = ARM_CP_CONST,
3373       .resetvalue = 0 },
3374     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
3375       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
3376       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3377     { .name = "VTCR_EL2", .state = ARM_CP_STATE_BOTH,
3378       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
3379       .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
3380       .type = ARM_CP_CONST, .resetvalue = 0 },
3381     { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
3382       .cp = 15, .opc1 = 6, .crm = 2,
3383       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3384       .type = ARM_CP_CONST | ARM_CP_64BIT, .resetvalue = 0 },
3385     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
3386       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
3387       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3388     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
3389       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
3390       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3391     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
3392       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
3393       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3394     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
3395       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
3396       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3397     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
3398       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
3399       .resetvalue = 0 },
3400     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
3401       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
3402       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3403     { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
3404       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
3405       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3406     { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
3407       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
3408       .resetvalue = 0 },
3409     { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
3410       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
3411       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3412     { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
3413       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
3414       .resetvalue = 0 },
3415     { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
3416       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
3417       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3418     { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
3419       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
3420       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3421     { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
3422       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
3423       .access = PL2_RW, .accessfn = access_tda,
3424       .type = ARM_CP_CONST, .resetvalue = 0 },
3425     { .name = "HPFAR_EL2", .state = ARM_CP_STATE_BOTH,
3426       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
3427       .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
3428       .type = ARM_CP_CONST, .resetvalue = 0 },
3429     REGINFO_SENTINEL
3430 };
3431
3432 static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
3433 {
3434     ARMCPU *cpu = arm_env_get_cpu(env);
3435     uint64_t valid_mask = HCR_MASK;
3436
3437     if (arm_feature(env, ARM_FEATURE_EL3)) {
3438         valid_mask &= ~HCR_HCD;
3439     } else {
3440         valid_mask &= ~HCR_TSC;
3441     }
3442
3443     /* Clear RES0 bits.  */
3444     value &= valid_mask;
3445
3446     /* These bits change the MMU setup:
3447      * HCR_VM enables stage 2 translation
3448      * HCR_PTW forbids certain page-table setups
3449      * HCR_DC Disables stage1 and enables stage2 translation
3450      */
3451     if ((raw_read(env, ri) ^ value) & (HCR_VM | HCR_PTW | HCR_DC)) {
3452         tlb_flush(CPU(cpu), 1);
3453     }
3454     raw_write(env, ri, value);
3455 }
3456
3457 static const ARMCPRegInfo el2_cp_reginfo[] = {
3458     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
3459       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
3460       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
3461       .writefn = hcr_write },
3462     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
3463       .type = ARM_CP_ALIAS,
3464       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
3465       .access = PL2_RW,
3466       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
3467     { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
3468       .type = ARM_CP_ALIAS,
3469       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
3470       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
3471     { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
3472       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
3473       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
3474     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
3475       .type = ARM_CP_ALIAS,
3476       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
3477       .access = PL2_RW,
3478       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_HYP]) },
3479     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
3480       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
3481       .access = PL2_RW, .writefn = vbar_write,
3482       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
3483       .resetvalue = 0 },
3484     { .name = "SP_EL2", .state = ARM_CP_STATE_AA64,
3485       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 1, .opc2 = 0,
3486       .access = PL3_RW, .type = ARM_CP_ALIAS,
3487       .fieldoffset = offsetof(CPUARMState, sp_el[2]) },
3488     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
3489       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
3490       .access = PL2_RW, .accessfn = cptr_access, .resetvalue = 0,
3491       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[2]) },
3492     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
3493       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
3494       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[2]),
3495       .resetvalue = 0 },
3496     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3497       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
3498       .access = PL2_RW, .type = ARM_CP_ALIAS,
3499       .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
3500     { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
3501       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
3502       .access = PL2_RW, .type = ARM_CP_CONST,
3503       .resetvalue = 0 },
3504     /* HAMAIR1 is mapped to AMAIR_EL2[63:32] */
3505     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3506       .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
3507       .access = PL2_RW, .type = ARM_CP_CONST,
3508       .resetvalue = 0 },
3509     { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
3510       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
3511       .access = PL2_RW, .type = ARM_CP_CONST,
3512       .resetvalue = 0 },
3513     { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
3514       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
3515       .access = PL2_RW, .type = ARM_CP_CONST,
3516       .resetvalue = 0 },
3517     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
3518       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
3519       .access = PL2_RW, .writefn = vmsa_tcr_el1_write,
3520       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
3521       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
3522     { .name = "VTCR", .state = ARM_CP_STATE_AA32,
3523       .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
3524       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3525       .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
3526     { .name = "VTCR_EL2", .state = ARM_CP_STATE_AA64,
3527       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
3528       .access = PL2_RW, .type = ARM_CP_ALIAS,
3529       .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
3530     { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
3531       .cp = 15, .opc1 = 6, .crm = 2,
3532       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
3533       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3534       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
3535       .writefn = vttbr_write },
3536     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
3537       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
3538       .access = PL2_RW, .writefn = vttbr_write,
3539       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
3540     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
3541       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
3542       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
3543       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[2]) },
3544     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
3545       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
3546       .access = PL2_RW, .resetvalue = 0,
3547       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) },
3548     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
3549       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
3550       .access = PL2_RW, .resetvalue = 0,
3551       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
3552     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
3553       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
3554       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
3555     { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
3556       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
3557       .type = ARM_CP_NO_RAW, .access = PL2_W,
3558       .writefn = tlbi_aa64_alle2_write },
3559     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
3560       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
3561       .type = ARM_CP_NO_RAW, .access = PL2_W,
3562       .writefn = tlbi_aa64_vae2_write },
3563     { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
3564       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
3565       .access = PL2_W, .type = ARM_CP_NO_RAW,
3566       .writefn = tlbi_aa64_vae2_write },
3567     { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
3568       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
3569       .access = PL2_W, .type = ARM_CP_NO_RAW,
3570       .writefn = tlbi_aa64_alle2is_write },
3571     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
3572       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
3573       .type = ARM_CP_NO_RAW, .access = PL2_W,
3574       .writefn = tlbi_aa64_vae2is_write },
3575     { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
3576       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
3577       .access = PL2_W, .type = ARM_CP_NO_RAW,
3578       .writefn = tlbi_aa64_vae2is_write },
3579 #ifndef CONFIG_USER_ONLY
3580     /* Unlike the other EL2-related AT operations, these must
3581      * UNDEF from EL3 if EL2 is not implemented, which is why we
3582      * define them here rather than with the rest of the AT ops.
3583      */
3584     { .name = "AT_S1E2R", .state = ARM_CP_STATE_AA64,
3585       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
3586       .access = PL2_W, .accessfn = at_s1e2_access,
3587       .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3588     { .name = "AT_S1E2W", .state = ARM_CP_STATE_AA64,
3589       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
3590       .access = PL2_W, .accessfn = at_s1e2_access,
3591       .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3592     /* The AArch32 ATS1H* operations are CONSTRAINED UNPREDICTABLE
3593      * if EL2 is not implemented; we choose to UNDEF. Behaviour at EL3
3594      * with SCR.NS == 0 outside Monitor mode is UNPREDICTABLE; we choose
3595      * to behave as if SCR.NS was 1.
3596      */
3597     { .name = "ATS1HR", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
3598       .access = PL2_W,
3599       .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
3600     { .name = "ATS1HW", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
3601       .access = PL2_W,
3602       .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
3603     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
3604       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
3605       /* ARMv7 requires bit 0 and 1 to reset to 1. ARMv8 defines the
3606        * reset values as IMPDEF. We choose to reset to 3 to comply with
3607        * both ARMv7 and ARMv8.
3608        */
3609       .access = PL2_RW, .resetvalue = 3,
3610       .fieldoffset = offsetof(CPUARMState, cp15.cnthctl_el2) },
3611     { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
3612       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
3613       .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 0,
3614       .writefn = gt_cntvoff_write,
3615       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
3616     { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
3617       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS | ARM_CP_IO,
3618       .writefn = gt_cntvoff_write,
3619       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
3620     { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
3621       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
3622       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
3623       .type = ARM_CP_IO, .access = PL2_RW,
3624       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
3625     { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
3626       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
3627       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_IO,
3628       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
3629     { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
3630       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
3631       .type = ARM_CP_IO, .access = PL2_RW,
3632       .resetfn = gt_hyp_timer_reset,
3633       .readfn = gt_hyp_tval_read, .writefn = gt_hyp_tval_write },
3634     { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
3635       .type = ARM_CP_IO,
3636       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
3637       .access = PL2_RW,
3638       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
3639       .resetvalue = 0,
3640       .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
3641 #endif
3642     /* The only field of MDCR_EL2 that has a defined architectural reset value
3643      * is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N; but we
3644      * don't impelment any PMU event counters, so using zero as a reset
3645      * value for MDCR_EL2 is okay
3646      */
3647     { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
3648       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
3649       .access = PL2_RW, .resetvalue = 0,
3650       .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2), },
3651     { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
3652       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
3653       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3654       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
3655     { .name = "HPFAR_EL2", .state = ARM_CP_STATE_AA64,
3656       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
3657       .access = PL2_RW,
3658       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
3659     REGINFO_SENTINEL
3660 };
3661
3662 static CPAccessResult nsacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
3663                                    bool isread)
3664 {
3665     /* The NSACR is RW at EL3, and RO for NS EL1 and NS EL2.
3666      * At Secure EL1 it traps to EL3.
3667      */
3668     if (arm_current_el(env) == 3) {
3669         return CP_ACCESS_OK;
3670     }
3671     if (arm_is_secure_below_el3(env)) {
3672         return CP_ACCESS_TRAP_EL3;
3673     }
3674     /* Accesses from EL1 NS and EL2 NS are UNDEF for write but allow reads. */
3675     if (isread) {
3676         return CP_ACCESS_OK;
3677     }
3678     return CP_ACCESS_TRAP_UNCATEGORIZED;
3679 }
3680
3681 static const ARMCPRegInfo el3_cp_reginfo[] = {
3682     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
3683       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
3684       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
3685       .resetvalue = 0, .writefn = scr_write },
3686     { .name = "SCR",  .type = ARM_CP_ALIAS,
3687       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
3688       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
3689       .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
3690       .writefn = scr_write },
3691     { .name = "MDCR_EL3", .state = ARM_CP_STATE_AA64,
3692       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 3, .opc2 = 1,
3693       .resetvalue = 0,
3694       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3) },
3695     { .name = "SDCR", .type = ARM_CP_ALIAS,
3696       .cp = 15, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 1,
3697       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
3698       .fieldoffset = offsetoflow32(CPUARMState, cp15.mdcr_el3) },
3699     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
3700       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
3701       .access = PL3_RW, .resetvalue = 0,
3702       .fieldoffset = offsetof(CPUARMState, cp15.sder) },
3703     { .name = "SDER",
3704       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
3705       .access = PL3_RW, .resetvalue = 0,
3706       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
3707     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
3708       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
3709       .writefn = vbar_write, .resetvalue = 0,
3710       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
3711     { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
3712       .type = ARM_CP_ALIAS, /* reset handled by AArch32 view */
3713       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
3714       .access = PL3_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
3715       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]) },
3716     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
3717       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
3718       .access = PL3_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
3719       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
3720     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
3721       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
3722       .access = PL3_RW, .writefn = vmsa_tcr_el1_write,
3723       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
3724       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
3725     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
3726       .type = ARM_CP_ALIAS,
3727       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
3728       .access = PL3_RW,
3729       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
3730     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
3731       .type = ARM_CP_ALIAS,
3732       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
3733       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
3734     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
3735       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
3736       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
3737     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
3738       .type = ARM_CP_ALIAS,
3739       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
3740       .access = PL3_RW,
3741       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_MON]) },
3742     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
3743       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
3744       .access = PL3_RW, .writefn = vbar_write,
3745       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
3746       .resetvalue = 0 },
3747     { .name = "CPTR_EL3", .state = ARM_CP_STATE_AA64,
3748       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
3749       .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
3750       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
3751     { .name = "TPIDR_EL3", .state = ARM_CP_STATE_AA64,
3752       .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 2,
3753       .access = PL3_RW, .resetvalue = 0,
3754       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[3]) },
3755     { .name = "AMAIR_EL3", .state = ARM_CP_STATE_AA64,
3756       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 3, .opc2 = 0,
3757       .access = PL3_RW, .type = ARM_CP_CONST,
3758       .resetvalue = 0 },
3759     { .name = "AFSR0_EL3", .state = ARM_CP_STATE_BOTH,
3760       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 0,
3761       .access = PL3_RW, .type = ARM_CP_CONST,
3762       .resetvalue = 0 },
3763     { .name = "AFSR1_EL3", .state = ARM_CP_STATE_BOTH,
3764       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 1,
3765       .access = PL3_RW, .type = ARM_CP_CONST,
3766       .resetvalue = 0 },
3767     { .name = "TLBI_ALLE3IS", .state = ARM_CP_STATE_AA64,
3768       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 0,
3769       .access = PL3_W, .type = ARM_CP_NO_RAW,
3770       .writefn = tlbi_aa64_alle3is_write },
3771     { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
3772       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
3773       .access = PL3_W, .type = ARM_CP_NO_RAW,
3774       .writefn = tlbi_aa64_vae3is_write },
3775     { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
3776       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
3777       .access = PL3_W, .type = ARM_CP_NO_RAW,
3778       .writefn = tlbi_aa64_vae3is_write },
3779     { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
3780       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
3781       .access = PL3_W, .type = ARM_CP_NO_RAW,
3782       .writefn = tlbi_aa64_alle3_write },
3783     { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
3784       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
3785       .access = PL3_W, .type = ARM_CP_NO_RAW,
3786       .writefn = tlbi_aa64_vae3_write },
3787     { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
3788       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
3789       .access = PL3_W, .type = ARM_CP_NO_RAW,
3790       .writefn = tlbi_aa64_vae3_write },
3791     REGINFO_SENTINEL
3792 };
3793
3794 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
3795                                      bool isread)
3796 {
3797     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
3798      * but the AArch32 CTR has its own reginfo struct)
3799      */
3800     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
3801         return CP_ACCESS_TRAP;
3802     }
3803     return CP_ACCESS_OK;
3804 }
3805
3806 static void oslar_write(CPUARMState *env, const ARMCPRegInfo *ri,
3807                         uint64_t value)
3808 {
3809     /* Writes to OSLAR_EL1 may update the OS lock status, which can be
3810      * read via a bit in OSLSR_EL1.
3811      */
3812     int oslock;
3813
3814     if (ri->state == ARM_CP_STATE_AA32) {
3815         oslock = (value == 0xC5ACCE55);
3816     } else {
3817         oslock = value & 1;
3818     }
3819
3820     env->cp15.oslsr_el1 = deposit32(env->cp15.oslsr_el1, 1, 1, oslock);
3821 }
3822
3823 static const ARMCPRegInfo debug_cp_reginfo[] = {
3824     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
3825      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
3826      * unlike DBGDRAR it is never accessible from EL0.
3827      * DBGDSAR is deprecated and must RAZ from v8 anyway, so it has no AArch64
3828      * accessor.
3829      */
3830     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
3831       .access = PL0_R, .accessfn = access_tdra,
3832       .type = ARM_CP_CONST, .resetvalue = 0 },
3833     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
3834       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
3835       .access = PL1_R, .accessfn = access_tdra,
3836       .type = ARM_CP_CONST, .resetvalue = 0 },
3837     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
3838       .access = PL0_R, .accessfn = access_tdra,
3839       .type = ARM_CP_CONST, .resetvalue = 0 },
3840     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
3841     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
3842       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
3843       .access = PL1_RW, .accessfn = access_tda,
3844       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
3845       .resetvalue = 0 },
3846     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
3847      * We don't implement the configurable EL0 access.
3848      */
3849     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
3850       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
3851       .type = ARM_CP_ALIAS,
3852       .access = PL1_R, .accessfn = access_tda,
3853       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1), },
3854     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
3855       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
3856       .access = PL1_W, .type = ARM_CP_NO_RAW,
3857       .accessfn = access_tdosa,
3858       .writefn = oslar_write },
3859     { .name = "OSLSR_EL1", .state = ARM_CP_STATE_BOTH,
3860       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 4,
3861       .access = PL1_R, .resetvalue = 10,
3862       .accessfn = access_tdosa,
3863       .fieldoffset = offsetof(CPUARMState, cp15.oslsr_el1) },
3864     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
3865     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
3866       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
3867       .access = PL1_RW, .accessfn = access_tdosa,
3868       .type = ARM_CP_NOP },
3869     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
3870      * implement vector catch debug events yet.
3871      */
3872     { .name = "DBGVCR",
3873       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
3874       .access = PL1_RW, .accessfn = access_tda,
3875       .type = ARM_CP_NOP },
3876     REGINFO_SENTINEL
3877 };
3878
3879 static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
3880     /* 64 bit access versions of the (dummy) debug registers */
3881     { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
3882       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
3883     { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
3884       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
3885     REGINFO_SENTINEL
3886 };
3887
3888 void hw_watchpoint_update(ARMCPU *cpu, int n)
3889 {
3890     CPUARMState *env = &cpu->env;
3891     vaddr len = 0;
3892     vaddr wvr = env->cp15.dbgwvr[n];
3893     uint64_t wcr = env->cp15.dbgwcr[n];
3894     int mask;
3895     int flags = BP_CPU | BP_STOP_BEFORE_ACCESS;
3896
3897     if (env->cpu_watchpoint[n]) {
3898         cpu_watchpoint_remove_by_ref(CPU(cpu), env->cpu_watchpoint[n]);
3899         env->cpu_watchpoint[n] = NULL;
3900     }
3901
3902     if (!extract64(wcr, 0, 1)) {
3903         /* E bit clear : watchpoint disabled */
3904         return;
3905     }
3906
3907     switch (extract64(wcr, 3, 2)) {
3908     case 0:
3909         /* LSC 00 is reserved and must behave as if the wp is disabled */
3910         return;
3911     case 1:
3912         flags |= BP_MEM_READ;
3913         break;
3914     case 2:
3915         flags |= BP_MEM_WRITE;
3916         break;
3917     case 3:
3918         flags |= BP_MEM_ACCESS;
3919         break;
3920     }
3921
3922     /* Attempts to use both MASK and BAS fields simultaneously are
3923      * CONSTRAINED UNPREDICTABLE; we opt to ignore BAS in this case,
3924      * thus generating a watchpoint for every byte in the masked region.
3925      */
3926     mask = extract64(wcr, 24, 4);
3927     if (mask == 1 || mask == 2) {
3928         /* Reserved values of MASK; we must act as if the mask value was
3929          * some non-reserved value, or as if the watchpoint were disabled.
3930          * We choose the latter.
3931          */
3932         return;
3933     } else if (mask) {
3934         /* Watchpoint covers an aligned area up to 2GB in size */
3935         len = 1ULL << mask;
3936         /* If masked bits in WVR are not zero it's CONSTRAINED UNPREDICTABLE
3937          * whether the watchpoint fires when the unmasked bits match; we opt
3938          * to generate the exceptions.
3939          */
3940         wvr &= ~(len - 1);
3941     } else {
3942         /* Watchpoint covers bytes defined by the byte address select bits */
3943         int bas = extract64(wcr, 5, 8);
3944         int basstart;
3945
3946         if (bas == 0) {
3947             /* This must act as if the watchpoint is disabled */
3948             return;
3949         }
3950
3951         if (extract64(wvr, 2, 1)) {
3952             /* Deprecated case of an only 4-aligned address. BAS[7:4] are
3953              * ignored, and BAS[3:0] define which bytes to watch.
3954              */
3955             bas &= 0xf;
3956         }
3957         /* The BAS bits are supposed to be programmed to indicate a contiguous
3958          * range of bytes. Otherwise it is CONSTRAINED UNPREDICTABLE whether
3959          * we fire for each byte in the word/doubleword addressed by the WVR.
3960          * We choose to ignore any non-zero bits after the first range of 1s.
3961          */
3962         basstart = ctz32(bas);
3963         len = cto32(bas >> basstart);
3964         wvr += basstart;
3965     }
3966
3967     cpu_watchpoint_insert(CPU(cpu), wvr, len, flags,
3968                           &env->cpu_watchpoint[n]);
3969 }
3970
3971 void hw_watchpoint_update_all(ARMCPU *cpu)
3972 {
3973     int i;
3974     CPUARMState *env = &cpu->env;
3975
3976     /* Completely clear out existing QEMU watchpoints and our array, to
3977      * avoid possible stale entries following migration load.
3978      */
3979     cpu_watchpoint_remove_all(CPU(cpu), BP_CPU);
3980     memset(env->cpu_watchpoint, 0, sizeof(env->cpu_watchpoint));
3981
3982     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_watchpoint); i++) {
3983         hw_watchpoint_update(cpu, i);
3984     }
3985 }
3986
3987 static void dbgwvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3988                          uint64_t value)
3989 {
3990     ARMCPU *cpu = arm_env_get_cpu(env);
3991     int i = ri->crm;
3992
3993     /* Bits [63:49] are hardwired to the value of bit [48]; that is, the
3994      * register reads and behaves as if values written are sign extended.
3995      * Bits [1:0] are RES0.
3996      */
3997     value = sextract64(value, 0, 49) & ~3ULL;
3998
3999     raw_write(env, ri, value);
4000     hw_watchpoint_update(cpu, i);
4001 }
4002
4003 static void dbgwcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4004                          uint64_t value)
4005 {
4006     ARMCPU *cpu = arm_env_get_cpu(env);
4007     int i = ri->crm;
4008
4009     raw_write(env, ri, value);
4010     hw_watchpoint_update(cpu, i);
4011 }
4012
4013 void hw_breakpoint_update(ARMCPU *cpu, int n)
4014 {
4015     CPUARMState *env = &cpu->env;
4016     uint64_t bvr = env->cp15.dbgbvr[n];
4017     uint64_t bcr = env->cp15.dbgbcr[n];
4018     vaddr addr;
4019     int bt;
4020     int flags = BP_CPU;
4021
4022     if (env->cpu_breakpoint[n]) {
4023         cpu_breakpoint_remove_by_ref(CPU(cpu), env->cpu_breakpoint[n]);
4024         env->cpu_breakpoint[n] = NULL;
4025     }
4026
4027     if (!extract64(bcr, 0, 1)) {
4028         /* E bit clear : watchpoint disabled */
4029         return;
4030     }
4031
4032     bt = extract64(bcr, 20, 4);
4033
4034     switch (bt) {
4035     case 4: /* unlinked address mismatch (reserved if AArch64) */
4036     case 5: /* linked address mismatch (reserved if AArch64) */
4037         qemu_log_mask(LOG_UNIMP,
4038                       "arm: address mismatch breakpoint types not implemented");
4039         return;
4040     case 0: /* unlinked address match */
4041     case 1: /* linked address match */
4042     {
4043         /* Bits [63:49] are hardwired to the value of bit [48]; that is,
4044          * we behave as if the register was sign extended. Bits [1:0] are
4045          * RES0. The BAS field is used to allow setting breakpoints on 16
4046          * bit wide instructions; it is CONSTRAINED UNPREDICTABLE whether
4047          * a bp will fire if the addresses covered by the bp and the addresses
4048          * covered by the insn overlap but the insn doesn't start at the
4049          * start of the bp address range. We choose to require the insn and
4050          * the bp to have the same address. The constraints on writing to
4051          * BAS enforced in dbgbcr_write mean we have only four cases:
4052          *  0b0000  => no breakpoint
4053          *  0b0011  => breakpoint on addr
4054          *  0b1100  => breakpoint on addr + 2
4055          *  0b1111  => breakpoint on addr
4056          * See also figure D2-3 in the v8 ARM ARM (DDI0487A.c).
4057          */
4058         int bas = extract64(bcr, 5, 4);
4059         addr = sextract64(bvr, 0, 49) & ~3ULL;
4060         if (bas == 0) {
4061             return;
4062         }
4063         if (bas == 0xc) {
4064             addr += 2;
4065         }
4066         break;
4067     }
4068     case 2: /* unlinked context ID match */
4069     case 8: /* unlinked VMID match (reserved if no EL2) */
4070     case 10: /* unlinked context ID and VMID match (reserved if no EL2) */
4071         qemu_log_mask(LOG_UNIMP,
4072                       "arm: unlinked context breakpoint types not implemented");
4073         return;
4074     case 9: /* linked VMID match (reserved if no EL2) */
4075     case 11: /* linked context ID and VMID match (reserved if no EL2) */
4076     case 3: /* linked context ID match */
4077     default:
4078         /* We must generate no events for Linked context matches (unless
4079          * they are linked to by some other bp/wp, which is handled in
4080          * updates for the linking bp/wp). We choose to also generate no events
4081          * for reserved values.
4082          */
4083         return;
4084     }
4085
4086     cpu_breakpoint_insert(CPU(cpu), addr, flags, &env->cpu_breakpoint[n]);
4087 }
4088
4089 void hw_breakpoint_update_all(ARMCPU *cpu)
4090 {
4091     int i;
4092     CPUARMState *env = &cpu->env;
4093
4094     /* Completely clear out existing QEMU breakpoints and our array, to
4095      * avoid possible stale entries following migration load.
4096      */
4097     cpu_breakpoint_remove_all(CPU(cpu), BP_CPU);
4098     memset(env->cpu_breakpoint, 0, sizeof(env->cpu_breakpoint));
4099
4100     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_breakpoint); i++) {
4101         hw_breakpoint_update(cpu, i);
4102     }
4103 }
4104
4105 static void dbgbvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4106                          uint64_t value)
4107 {
4108     ARMCPU *cpu = arm_env_get_cpu(env);
4109     int i = ri->crm;
4110
4111     raw_write(env, ri, value);
4112     hw_breakpoint_update(cpu, i);
4113 }
4114
4115 static void dbgbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4116                          uint64_t value)
4117 {
4118     ARMCPU *cpu = arm_env_get_cpu(env);
4119     int i = ri->crm;
4120
4121     /* BAS[3] is a read-only copy of BAS[2], and BAS[1] a read-only
4122      * copy of BAS[0].
4123      */
4124     value = deposit64(value, 6, 1, extract64(value, 5, 1));
4125     value = deposit64(value, 8, 1, extract64(value, 7, 1));
4126
4127     raw_write(env, ri, value);
4128     hw_breakpoint_update(cpu, i);
4129 }
4130
4131 static void define_debug_regs(ARMCPU *cpu)
4132 {
4133     /* Define v7 and v8 architectural debug registers.
4134      * These are just dummy implementations for now.
4135      */
4136     int i;
4137     int wrps, brps, ctx_cmps;
4138     ARMCPRegInfo dbgdidr = {
4139         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
4140         .access = PL0_R, .accessfn = access_tda,
4141         .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
4142     };
4143
4144     /* Note that all these register fields hold "number of Xs minus 1". */
4145     brps = extract32(cpu->dbgdidr, 24, 4);
4146     wrps = extract32(cpu->dbgdidr, 28, 4);
4147     ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
4148
4149     assert(ctx_cmps <= brps);
4150
4151     /* The DBGDIDR and ID_AA64DFR0_EL1 define various properties
4152      * of the debug registers such as number of breakpoints;
4153      * check that if they both exist then they agree.
4154      */
4155     if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
4156         assert(extract32(cpu->id_aa64dfr0, 12, 4) == brps);
4157         assert(extract32(cpu->id_aa64dfr0, 20, 4) == wrps);
4158         assert(extract32(cpu->id_aa64dfr0, 28, 4) == ctx_cmps);
4159     }
4160
4161     define_one_arm_cp_reg(cpu, &dbgdidr);
4162     define_arm_cp_regs(cpu, debug_cp_reginfo);
4163
4164     if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
4165         define_arm_cp_regs(cpu, debug_lpae_cp_reginfo);
4166     }
4167
4168     for (i = 0; i < brps + 1; i++) {
4169         ARMCPRegInfo dbgregs[] = {
4170             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
4171               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
4172               .access = PL1_RW, .accessfn = access_tda,
4173               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
4174               .writefn = dbgbvr_write, .raw_writefn = raw_write
4175             },
4176             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
4177               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
4178               .access = PL1_RW, .accessfn = access_tda,
4179               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
4180               .writefn = dbgbcr_write, .raw_writefn = raw_write
4181             },
4182             REGINFO_SENTINEL
4183         };
4184         define_arm_cp_regs(cpu, dbgregs);
4185     }
4186
4187     for (i = 0; i < wrps + 1; i++) {
4188         ARMCPRegInfo dbgregs[] = {
4189             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
4190               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
4191               .access = PL1_RW, .accessfn = access_tda,
4192               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
4193               .writefn = dbgwvr_write, .raw_writefn = raw_write
4194             },
4195             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
4196               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
4197               .access = PL1_RW, .accessfn = access_tda,
4198               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
4199               .writefn = dbgwcr_write, .raw_writefn = raw_write
4200             },
4201             REGINFO_SENTINEL
4202         };
4203         define_arm_cp_regs(cpu, dbgregs);
4204     }
4205 }
4206
4207 void register_cp_regs_for_features(ARMCPU *cpu)
4208 {
4209     /* Register all the coprocessor registers based on feature bits */
4210     CPUARMState *env = &cpu->env;
4211     if (arm_feature(env, ARM_FEATURE_M)) {
4212         /* M profile has no coprocessor registers */
4213         return;
4214     }
4215
4216     define_arm_cp_regs(cpu, cp_reginfo);
4217     if (!arm_feature(env, ARM_FEATURE_V8)) {
4218         /* Must go early as it is full of wildcards that may be
4219          * overridden by later definitions.
4220          */
4221         define_arm_cp_regs(cpu, not_v8_cp_reginfo);
4222     }
4223
4224     if (arm_feature(env, ARM_FEATURE_V6)) {
4225         /* The ID registers all have impdef reset values */
4226         ARMCPRegInfo v6_idregs[] = {
4227             { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
4228               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
4229               .access = PL1_R, .type = ARM_CP_CONST,
4230               .resetvalue = cpu->id_pfr0 },
4231             { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
4232               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
4233               .access = PL1_R, .type = ARM_CP_CONST,
4234               .resetvalue = cpu->id_pfr1 },
4235             { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
4236               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
4237               .access = PL1_R, .type = ARM_CP_CONST,
4238               .resetvalue = cpu->id_dfr0 },
4239             { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
4240               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
4241               .access = PL1_R, .type = ARM_CP_CONST,
4242               .resetvalue = cpu->id_afr0 },
4243             { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
4244               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
4245               .access = PL1_R, .type = ARM_CP_CONST,
4246               .resetvalue = cpu->id_mmfr0 },
4247             { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
4248               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
4249               .access = PL1_R, .type = ARM_CP_CONST,
4250               .resetvalue = cpu->id_mmfr1 },
4251             { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
4252               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
4253               .access = PL1_R, .type = ARM_CP_CONST,
4254               .resetvalue = cpu->id_mmfr2 },
4255             { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
4256               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
4257               .access = PL1_R, .type = ARM_CP_CONST,
4258               .resetvalue = cpu->id_mmfr3 },
4259             { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
4260               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
4261               .access = PL1_R, .type = ARM_CP_CONST,
4262               .resetvalue = cpu->id_isar0 },
4263             { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
4264               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
4265               .access = PL1_R, .type = ARM_CP_CONST,
4266               .resetvalue = cpu->id_isar1 },
4267             { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
4268               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
4269               .access = PL1_R, .type = ARM_CP_CONST,
4270               .resetvalue = cpu->id_isar2 },
4271             { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
4272               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
4273               .access = PL1_R, .type = ARM_CP_CONST,
4274               .resetvalue = cpu->id_isar3 },
4275             { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
4276               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
4277               .access = PL1_R, .type = ARM_CP_CONST,
4278               .resetvalue = cpu->id_isar4 },
4279             { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
4280               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
4281               .access = PL1_R, .type = ARM_CP_CONST,
4282               .resetvalue = cpu->id_isar5 },
4283             /* 6..7 are as yet unallocated and must RAZ */
4284             { .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
4285               .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
4286               .resetvalue = 0 },
4287             { .name = "ID_ISAR7", .cp = 15, .crn = 0, .crm = 2,
4288               .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
4289               .resetvalue = 0 },
4290             REGINFO_SENTINEL
4291         };
4292         define_arm_cp_regs(cpu, v6_idregs);
4293         define_arm_cp_regs(cpu, v6_cp_reginfo);
4294     } else {
4295         define_arm_cp_regs(cpu, not_v6_cp_reginfo);
4296     }
4297     if (arm_feature(env, ARM_FEATURE_V6K)) {
4298         define_arm_cp_regs(cpu, v6k_cp_reginfo);
4299     }
4300     if (arm_feature(env, ARM_FEATURE_V7MP) &&
4301         !arm_feature(env, ARM_FEATURE_MPU)) {
4302         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
4303     }
4304     if (arm_feature(env, ARM_FEATURE_V7)) {
4305         /* v7 performance monitor control register: same implementor
4306          * field as main ID register, and we implement only the cycle
4307          * count register.
4308          */
4309 #ifndef CONFIG_USER_ONLY
4310         ARMCPRegInfo pmcr = {
4311             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
4312             .access = PL0_RW,
4313             .type = ARM_CP_IO | ARM_CP_ALIAS,
4314             .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
4315             .accessfn = pmreg_access, .writefn = pmcr_write,
4316             .raw_writefn = raw_write,
4317         };
4318         ARMCPRegInfo pmcr64 = {
4319             .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
4320             .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
4321             .access = PL0_RW, .accessfn = pmreg_access,
4322             .type = ARM_CP_IO,
4323             .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
4324             .resetvalue = cpu->midr & 0xff000000,
4325             .writefn = pmcr_write, .raw_writefn = raw_write,
4326         };
4327         define_one_arm_cp_reg(cpu, &pmcr);
4328         define_one_arm_cp_reg(cpu, &pmcr64);
4329 #endif
4330         ARMCPRegInfo clidr = {
4331             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
4332             .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
4333             .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->clidr
4334         };
4335         define_one_arm_cp_reg(cpu, &clidr);
4336         define_arm_cp_regs(cpu, v7_cp_reginfo);
4337         define_debug_regs(cpu);
4338     } else {
4339         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
4340     }
4341     if (arm_feature(env, ARM_FEATURE_V8)) {
4342         /* AArch64 ID registers, which all have impdef reset values */
4343         ARMCPRegInfo v8_idregs[] = {
4344             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
4345               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
4346               .access = PL1_R, .type = ARM_CP_CONST,
4347               .resetvalue = cpu->id_aa64pfr0 },
4348             { .name = "ID_AA64PFR1_EL1", .state = ARM_CP_STATE_AA64,
4349               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
4350               .access = PL1_R, .type = ARM_CP_CONST,
4351               .resetvalue = cpu->id_aa64pfr1},
4352             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
4353               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
4354               .access = PL1_R, .type = ARM_CP_CONST,
4355               /* We mask out the PMUVer field, because we don't currently
4356                * implement the PMU. Not advertising it prevents the guest
4357                * from trying to use it and getting UNDEFs on registers we
4358                * don't implement.
4359                */
4360               .resetvalue = cpu->id_aa64dfr0 & ~0xf00 },
4361             { .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
4362               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
4363               .access = PL1_R, .type = ARM_CP_CONST,
4364               .resetvalue = cpu->id_aa64dfr1 },
4365             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
4366               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
4367               .access = PL1_R, .type = ARM_CP_CONST,
4368               .resetvalue = cpu->id_aa64afr0 },
4369             { .name = "ID_AA64AFR1_EL1", .state = ARM_CP_STATE_AA64,
4370               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
4371               .access = PL1_R, .type = ARM_CP_CONST,
4372               .resetvalue = cpu->id_aa64afr1 },
4373             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
4374               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
4375               .access = PL1_R, .type = ARM_CP_CONST,
4376               .resetvalue = cpu->id_aa64isar0 },
4377             { .name = "ID_AA64ISAR1_EL1", .state = ARM_CP_STATE_AA64,
4378               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
4379               .access = PL1_R, .type = ARM_CP_CONST,
4380               .resetvalue = cpu->id_aa64isar1 },
4381             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
4382               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
4383               .access = PL1_R, .type = ARM_CP_CONST,
4384               .resetvalue = cpu->id_aa64mmfr0 },
4385             { .name = "ID_AA64MMFR1_EL1", .state = ARM_CP_STATE_AA64,
4386               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
4387               .access = PL1_R, .type = ARM_CP_CONST,
4388               .resetvalue = cpu->id_aa64mmfr1 },
4389             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
4390               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
4391               .access = PL1_R, .type = ARM_CP_CONST,
4392               .resetvalue = cpu->mvfr0 },
4393             { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
4394               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
4395               .access = PL1_R, .type = ARM_CP_CONST,
4396               .resetvalue = cpu->mvfr1 },
4397             { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
4398               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
4399               .access = PL1_R, .type = ARM_CP_CONST,
4400               .resetvalue = cpu->mvfr2 },
4401             { .name = "PMCEID0", .state = ARM_CP_STATE_AA32,
4402               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 6,
4403               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4404               .resetvalue = cpu->pmceid0 },
4405             { .name = "PMCEID0_EL0", .state = ARM_CP_STATE_AA64,
4406               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 6,
4407               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4408               .resetvalue = cpu->pmceid0 },
4409             { .name = "PMCEID1", .state = ARM_CP_STATE_AA32,
4410               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 7,
4411               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4412               .resetvalue = cpu->pmceid1 },
4413             { .name = "PMCEID1_EL0", .state = ARM_CP_STATE_AA64,
4414               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 7,
4415               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4416               .resetvalue = cpu->pmceid1 },
4417             REGINFO_SENTINEL
4418         };
4419         /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
4420         if (!arm_feature(env, ARM_FEATURE_EL3) &&
4421             !arm_feature(env, ARM_FEATURE_EL2)) {
4422             ARMCPRegInfo rvbar = {
4423                 .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
4424                 .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
4425                 .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
4426             };
4427             define_one_arm_cp_reg(cpu, &rvbar);
4428         }
4429         define_arm_cp_regs(cpu, v8_idregs);
4430         define_arm_cp_regs(cpu, v8_cp_reginfo);
4431     }
4432     if (arm_feature(env, ARM_FEATURE_EL2)) {
4433         uint64_t vmpidr_def = mpidr_read_val(env);
4434         ARMCPRegInfo vpidr_regs[] = {
4435             { .name = "VPIDR", .state = ARM_CP_STATE_AA32,
4436               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
4437               .access = PL2_RW, .accessfn = access_el3_aa32ns,
4438               .resetvalue = cpu->midr,
4439               .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
4440             { .name = "VPIDR_EL2", .state = ARM_CP_STATE_AA64,
4441               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
4442               .access = PL2_RW, .resetvalue = cpu->midr,
4443               .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
4444             { .name = "VMPIDR", .state = ARM_CP_STATE_AA32,
4445               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
4446               .access = PL2_RW, .accessfn = access_el3_aa32ns,
4447               .resetvalue = vmpidr_def,
4448               .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
4449             { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_AA64,
4450               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
4451               .access = PL2_RW,
4452               .resetvalue = vmpidr_def,
4453               .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
4454             REGINFO_SENTINEL
4455         };
4456         define_arm_cp_regs(cpu, vpidr_regs);
4457         define_arm_cp_regs(cpu, el2_cp_reginfo);
4458         /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
4459         if (!arm_feature(env, ARM_FEATURE_EL3)) {
4460             ARMCPRegInfo rvbar = {
4461                 .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
4462                 .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
4463                 .type = ARM_CP_CONST, .access = PL2_R, .resetvalue = cpu->rvbar
4464             };
4465             define_one_arm_cp_reg(cpu, &rvbar);
4466         }
4467     } else {
4468         /* If EL2 is missing but higher ELs are enabled, we need to
4469          * register the no_el2 reginfos.
4470          */
4471         if (arm_feature(env, ARM_FEATURE_EL3)) {
4472             /* When EL3 exists but not EL2, VPIDR and VMPIDR take the value
4473              * of MIDR_EL1 and MPIDR_EL1.
4474              */
4475             ARMCPRegInfo vpidr_regs[] = {
4476                 { .name = "VPIDR_EL2", .state = ARM_CP_STATE_BOTH,
4477                   .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
4478                   .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
4479                   .type = ARM_CP_CONST, .resetvalue = cpu->midr,
4480                   .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
4481                 { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_BOTH,
4482                   .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
4483                   .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
4484                   .type = ARM_CP_NO_RAW,
4485                   .writefn = arm_cp_write_ignore, .readfn = mpidr_read },
4486                 REGINFO_SENTINEL
4487             };
4488             define_arm_cp_regs(cpu, vpidr_regs);
4489             define_arm_cp_regs(cpu, el3_no_el2_cp_reginfo);
4490         }
4491     }
4492     if (arm_feature(env, ARM_FEATURE_EL3)) {
4493         define_arm_cp_regs(cpu, el3_cp_reginfo);
4494         ARMCPRegInfo rvbar = {
4495             .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
4496             .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
4497             .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar
4498         };
4499         define_one_arm_cp_reg(cpu, &rvbar);
4500     }
4501     /* The behaviour of NSACR is sufficiently various that we don't
4502      * try to describe it in a single reginfo:
4503      *  if EL3 is 64 bit, then trap to EL3 from S EL1,
4504      *     reads as constant 0xc00 from NS EL1 and NS EL2
4505      *  if EL3 is 32 bit, then RW at EL3, RO at NS EL1 and NS EL2
4506      *  if v7 without EL3, register doesn't exist
4507      *  if v8 without EL3, reads as constant 0xc00 from NS EL1 and NS EL2
4508      */
4509     if (arm_feature(env, ARM_FEATURE_EL3)) {
4510         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
4511             ARMCPRegInfo nsacr = {
4512                 .name = "NSACR", .type = ARM_CP_CONST,
4513                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
4514                 .access = PL1_RW, .accessfn = nsacr_access,
4515                 .resetvalue = 0xc00
4516             };
4517             define_one_arm_cp_reg(cpu, &nsacr);
4518         } else {
4519             ARMCPRegInfo nsacr = {
4520                 .name = "NSACR",
4521                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
4522                 .access = PL3_RW | PL1_R,
4523                 .resetvalue = 0,
4524                 .fieldoffset = offsetof(CPUARMState, cp15.nsacr)
4525             };
4526             define_one_arm_cp_reg(cpu, &nsacr);
4527         }
4528     } else {
4529         if (arm_feature(env, ARM_FEATURE_V8)) {
4530             ARMCPRegInfo nsacr = {
4531                 .name = "NSACR", .type = ARM_CP_CONST,
4532                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
4533                 .access = PL1_R,
4534                 .resetvalue = 0xc00
4535             };
4536             define_one_arm_cp_reg(cpu, &nsacr);
4537         }
4538     }
4539
4540     if (arm_feature(env, ARM_FEATURE_MPU)) {
4541         if (arm_feature(env, ARM_FEATURE_V6)) {
4542             /* PMSAv6 not implemented */
4543             assert(arm_feature(env, ARM_FEATURE_V7));
4544             define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
4545             define_arm_cp_regs(cpu, pmsav7_cp_reginfo);
4546         } else {
4547             define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
4548         }
4549     } else {
4550         define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
4551         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
4552     }
4553     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
4554         define_arm_cp_regs(cpu, t2ee_cp_reginfo);
4555     }
4556     if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
4557         define_arm_cp_regs(cpu, generic_timer_cp_reginfo);
4558     }
4559     if (arm_feature(env, ARM_FEATURE_VAPA)) {
4560         define_arm_cp_regs(cpu, vapa_cp_reginfo);
4561     }
4562     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
4563         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
4564     }
4565     if (arm_feature(env, ARM_FEATURE_CACHE_DIRTY_REG)) {
4566         define_arm_cp_regs(cpu, cache_dirty_status_cp_reginfo);
4567     }
4568     if (arm_feature(env, ARM_FEATURE_CACHE_BLOCK_OPS)) {
4569         define_arm_cp_regs(cpu, cache_block_ops_cp_reginfo);
4570     }
4571     if (arm_feature(env, ARM_FEATURE_OMAPCP)) {
4572         define_arm_cp_regs(cpu, omap_cp_reginfo);
4573     }
4574     if (arm_feature(env, ARM_FEATURE_STRONGARM)) {
4575         define_arm_cp_regs(cpu, strongarm_cp_reginfo);
4576     }
4577     if (arm_feature(env, ARM_FEATURE_XSCALE)) {
4578         define_arm_cp_regs(cpu, xscale_cp_reginfo);
4579     }
4580     if (arm_feature(env, ARM_FEATURE_DUMMY_C15_REGS)) {
4581         define_arm_cp_regs(cpu, dummy_c15_cp_reginfo);
4582     }
4583     if (arm_feature(env, ARM_FEATURE_LPAE)) {
4584         define_arm_cp_regs(cpu, lpae_cp_reginfo);
4585     }
4586     /* Slightly awkwardly, the OMAP and StrongARM cores need all of
4587      * cp15 crn=0 to be writes-ignored, whereas for other cores they should
4588      * be read-only (ie write causes UNDEF exception).
4589      */
4590     {
4591         ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
4592             /* Pre-v8 MIDR space.
4593              * Note that the MIDR isn't a simple constant register because
4594              * of the TI925 behaviour where writes to another register can
4595              * cause the MIDR value to change.
4596              *
4597              * Unimplemented registers in the c15 0 0 0 space default to
4598              * MIDR. Define MIDR first as this entire space, then CTR, TCMTR
4599              * and friends override accordingly.
4600              */
4601             { .name = "MIDR",
4602               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
4603               .access = PL1_R, .resetvalue = cpu->midr,
4604               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
4605               .readfn = midr_read,
4606               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
4607               .type = ARM_CP_OVERRIDE },
4608             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
4609             { .name = "DUMMY",
4610               .cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
4611               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4612             { .name = "DUMMY",
4613               .cp = 15, .crn = 0, .crm = 4, .opc1 = 0, .opc2 = CP_ANY,
4614               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4615             { .name = "DUMMY",
4616               .cp = 15, .crn = 0, .crm = 5, .opc1 = 0, .opc2 = CP_ANY,
4617               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4618             { .name = "DUMMY",
4619               .cp = 15, .crn = 0, .crm = 6, .opc1 = 0, .opc2 = CP_ANY,
4620               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4621             { .name = "DUMMY",
4622               .cp = 15, .crn = 0, .crm = 7, .opc1 = 0, .opc2 = CP_ANY,
4623               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4624             REGINFO_SENTINEL
4625         };
4626         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
4627             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
4628               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
4629               .access = PL1_R, .type = ARM_CP_NO_RAW, .resetvalue = cpu->midr,
4630               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
4631               .readfn = midr_read },
4632             /* crn = 0 op1 = 0 crm = 0 op2 = 4,7 : AArch32 aliases of MIDR */
4633             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
4634               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
4635               .access = PL1_R, .resetvalue = cpu->midr },
4636             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
4637               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 7,
4638               .access = PL1_R, .resetvalue = cpu->midr },
4639             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
4640               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
4641               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->revidr },
4642             REGINFO_SENTINEL
4643         };
4644         ARMCPRegInfo id_cp_reginfo[] = {
4645             /* These are common to v8 and pre-v8 */
4646             { .name = "CTR",
4647               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
4648               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
4649             { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
4650               .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
4651               .access = PL0_R, .accessfn = ctr_el0_access,
4652               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
4653             /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
4654             { .name = "TCMTR",
4655               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
4656               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4657             REGINFO_SENTINEL
4658         };
4659         /* TLBTR is specific to VMSA */
4660         ARMCPRegInfo id_tlbtr_reginfo = {
4661               .name = "TLBTR",
4662               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
4663               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0,
4664         };
4665         /* MPUIR is specific to PMSA V6+ */
4666         ARMCPRegInfo id_mpuir_reginfo = {
4667               .name = "MPUIR",
4668               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
4669               .access = PL1_R, .type = ARM_CP_CONST,
4670               .resetvalue = cpu->pmsav7_dregion << 8
4671         };
4672         ARMCPRegInfo crn0_wi_reginfo = {
4673             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
4674             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
4675             .type = ARM_CP_NOP | ARM_CP_OVERRIDE
4676         };
4677         if (arm_feature(env, ARM_FEATURE_OMAPCP) ||
4678             arm_feature(env, ARM_FEATURE_STRONGARM)) {
4679             ARMCPRegInfo *r;
4680             /* Register the blanket "writes ignored" value first to cover the
4681              * whole space. Then update the specific ID registers to allow write
4682              * access, so that they ignore writes rather than causing them to
4683              * UNDEF.
4684              */
4685             define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
4686             for (r = id_pre_v8_midr_cp_reginfo;
4687                  r->type != ARM_CP_SENTINEL; r++) {
4688                 r->access = PL1_RW;
4689             }
4690             for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
4691                 r->access = PL1_RW;
4692             }
4693             id_tlbtr_reginfo.access = PL1_RW;
4694             id_tlbtr_reginfo.access = PL1_RW;
4695         }
4696         if (arm_feature(env, ARM_FEATURE_V8)) {
4697             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
4698         } else {
4699             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
4700         }
4701         define_arm_cp_regs(cpu, id_cp_reginfo);
4702         if (!arm_feature(env, ARM_FEATURE_MPU)) {
4703             define_one_arm_cp_reg(cpu, &id_tlbtr_reginfo);
4704         } else if (arm_feature(env, ARM_FEATURE_V7)) {
4705             define_one_arm_cp_reg(cpu, &id_mpuir_reginfo);
4706         }
4707     }
4708
4709     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
4710         define_arm_cp_regs(cpu, mpidr_cp_reginfo);
4711     }
4712
4713     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
4714         ARMCPRegInfo auxcr_reginfo[] = {
4715             { .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
4716               .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
4717               .access = PL1_RW, .type = ARM_CP_CONST,
4718               .resetvalue = cpu->reset_auxcr },
4719             { .name = "ACTLR_EL2", .state = ARM_CP_STATE_BOTH,
4720               .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 1,
4721               .access = PL2_RW, .type = ARM_CP_CONST,
4722               .resetvalue = 0 },
4723             { .name = "ACTLR_EL3", .state = ARM_CP_STATE_AA64,
4724               .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 1,
4725               .access = PL3_RW, .type = ARM_CP_CONST,
4726               .resetvalue = 0 },
4727             REGINFO_SENTINEL
4728         };
4729         define_arm_cp_regs(cpu, auxcr_reginfo);
4730     }
4731
4732     if (arm_feature(env, ARM_FEATURE_CBAR)) {
4733         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
4734             /* 32 bit view is [31:18] 0...0 [43:32]. */
4735             uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
4736                 | extract64(cpu->reset_cbar, 32, 12);
4737             ARMCPRegInfo cbar_reginfo[] = {
4738                 { .name = "CBAR",
4739                   .type = ARM_CP_CONST,
4740                   .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
4741                   .access = PL1_R, .resetvalue = cpu->reset_cbar },
4742                 { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
4743                   .type = ARM_CP_CONST,
4744                   .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
4745                   .access = PL1_R, .resetvalue = cbar32 },
4746                 REGINFO_SENTINEL
4747             };
4748             /* We don't implement a r/w 64 bit CBAR currently */
4749             assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
4750             define_arm_cp_regs(cpu, cbar_reginfo);
4751         } else {
4752             ARMCPRegInfo cbar = {
4753                 .name = "CBAR",
4754                 .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
4755                 .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
4756                 .fieldoffset = offsetof(CPUARMState,
4757                                         cp15.c15_config_base_address)
4758             };
4759             if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
4760                 cbar.access = PL1_R;
4761                 cbar.fieldoffset = 0;
4762                 cbar.type = ARM_CP_CONST;
4763             }
4764             define_one_arm_cp_reg(cpu, &cbar);
4765         }
4766     }
4767
4768     /* Generic registers whose values depend on the implementation */
4769     {
4770         ARMCPRegInfo sctlr = {
4771             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
4772             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
4773             .access = PL1_RW,
4774             .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
4775                                    offsetof(CPUARMState, cp15.sctlr_ns) },
4776             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
4777             .raw_writefn = raw_write,
4778         };
4779         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
4780             /* Normally we would always end the TB on an SCTLR write, but Linux
4781              * arch/arm/mach-pxa/sleep.S expects two instructions following
4782              * an MMU enable to execute from cache.  Imitate this behaviour.
4783              */
4784             sctlr.type |= ARM_CP_SUPPRESS_TB_END;
4785         }
4786         define_one_arm_cp_reg(cpu, &sctlr);
4787     }
4788 }
4789
4790 ARMCPU *cpu_arm_init(const char *cpu_model)
4791 {
4792     return ARM_CPU(cpu_generic_init(TYPE_ARM_CPU, cpu_model));
4793 }
4794
4795 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
4796 {
4797     CPUState *cs = CPU(cpu);
4798     CPUARMState *env = &cpu->env;
4799
4800     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
4801         gdb_register_coprocessor(cs, aarch64_fpu_gdb_get_reg,
4802                                  aarch64_fpu_gdb_set_reg,
4803                                  34, "aarch64-fpu.xml", 0);
4804     } else if (arm_feature(env, ARM_FEATURE_NEON)) {
4805         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
4806                                  51, "arm-neon.xml", 0);
4807     } else if (arm_feature(env, ARM_FEATURE_VFP3)) {
4808         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
4809                                  35, "arm-vfp3.xml", 0);
4810     } else if (arm_feature(env, ARM_FEATURE_VFP)) {
4811         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
4812                                  19, "arm-vfp.xml", 0);
4813     }
4814 }
4815
4816 /* Sort alphabetically by type name, except for "any". */
4817 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
4818 {
4819     ObjectClass *class_a = (ObjectClass *)a;
4820     ObjectClass *class_b = (ObjectClass *)b;
4821     const char *name_a, *name_b;
4822
4823     name_a = object_class_get_name(class_a);
4824     name_b = object_class_get_name(class_b);
4825     if (strcmp(name_a, "any-" TYPE_ARM_CPU) == 0) {
4826         return 1;
4827     } else if (strcmp(name_b, "any-" TYPE_ARM_CPU) == 0) {
4828         return -1;
4829     } else {
4830         return strcmp(name_a, name_b);
4831     }
4832 }
4833
4834 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
4835 {
4836     ObjectClass *oc = data;
4837     CPUListState *s = user_data;
4838     const char *typename;
4839     char *name;
4840
4841     typename = object_class_get_name(oc);
4842     name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_ARM_CPU));
4843     (*s->cpu_fprintf)(s->file, "  %s\n",
4844                       name);
4845     g_free(name);
4846 }
4847
4848 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
4849 {
4850     CPUListState s = {
4851         .file = f,
4852         .cpu_fprintf = cpu_fprintf,
4853     };
4854     GSList *list;
4855
4856     list = object_class_get_list(TYPE_ARM_CPU, false);
4857     list = g_slist_sort(list, arm_cpu_list_compare);
4858     (*cpu_fprintf)(f, "Available CPUs:\n");
4859     g_slist_foreach(list, arm_cpu_list_entry, &s);
4860     g_slist_free(list);
4861 #ifdef CONFIG_KVM
4862     /* The 'host' CPU type is dynamically registered only if KVM is
4863      * enabled, so we have to special-case it here:
4864      */
4865     (*cpu_fprintf)(f, "  host (only available in KVM mode)\n");
4866 #endif
4867 }
4868
4869 static void arm_cpu_add_definition(gpointer data, gpointer user_data)
4870 {
4871     ObjectClass *oc = data;
4872     CpuDefinitionInfoList **cpu_list = user_data;
4873     CpuDefinitionInfoList *entry;
4874     CpuDefinitionInfo *info;
4875     const char *typename;
4876
4877     typename = object_class_get_name(oc);
4878     info = g_malloc0(sizeof(*info));
4879     info->name = g_strndup(typename,
4880                            strlen(typename) - strlen("-" TYPE_ARM_CPU));
4881
4882     entry = g_malloc0(sizeof(*entry));
4883     entry->value = info;
4884     entry->next = *cpu_list;
4885     *cpu_list = entry;
4886 }
4887
4888 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
4889 {
4890     CpuDefinitionInfoList *cpu_list = NULL;
4891     GSList *list;
4892
4893     list = object_class_get_list(TYPE_ARM_CPU, false);
4894     g_slist_foreach(list, arm_cpu_add_definition, &cpu_list);
4895     g_slist_free(list);
4896
4897     return cpu_list;
4898 }
4899
4900 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
4901                                    void *opaque, int state, int secstate,
4902                                    int crm, int opc1, int opc2)
4903 {
4904     /* Private utility function for define_one_arm_cp_reg_with_opaque():
4905      * add a single reginfo struct to the hash table.
4906      */
4907     uint32_t *key = g_new(uint32_t, 1);
4908     ARMCPRegInfo *r2 = g_memdup(r, sizeof(ARMCPRegInfo));
4909     int is64 = (r->type & ARM_CP_64BIT) ? 1 : 0;
4910     int ns = (secstate & ARM_CP_SECSTATE_NS) ? 1 : 0;
4911
4912     /* Reset the secure state to the specific incoming state.  This is
4913      * necessary as the register may have been defined with both states.
4914      */
4915     r2->secure = secstate;
4916
4917     if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
4918         /* Register is banked (using both entries in array).
4919          * Overwriting fieldoffset as the array is only used to define
4920          * banked registers but later only fieldoffset is used.
4921          */
4922         r2->fieldoffset = r->bank_fieldoffsets[ns];
4923     }
4924
4925     if (state == ARM_CP_STATE_AA32) {
4926         if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
4927             /* If the register is banked then we don't need to migrate or
4928              * reset the 32-bit instance in certain cases:
4929              *
4930              * 1) If the register has both 32-bit and 64-bit instances then we
4931              *    can count on the 64-bit instance taking care of the
4932              *    non-secure bank.
4933              * 2) If ARMv8 is enabled then we can count on a 64-bit version
4934              *    taking care of the secure bank.  This requires that separate
4935              *    32 and 64-bit definitions are provided.
4936              */
4937             if ((r->state == ARM_CP_STATE_BOTH && ns) ||
4938                 (arm_feature(&cpu->env, ARM_FEATURE_V8) && !ns)) {
4939                 r2->type |= ARM_CP_ALIAS;
4940             }
4941         } else if ((secstate != r->secure) && !ns) {
4942             /* The register is not banked so we only want to allow migration of
4943              * the non-secure instance.
4944              */
4945             r2->type |= ARM_CP_ALIAS;
4946         }
4947
4948         if (r->state == ARM_CP_STATE_BOTH) {
4949             /* We assume it is a cp15 register if the .cp field is left unset.
4950              */
4951             if (r2->cp == 0) {
4952                 r2->cp = 15;
4953             }
4954
4955 #ifdef HOST_WORDS_BIGENDIAN
4956             if (r2->fieldoffset) {
4957                 r2->fieldoffset += sizeof(uint32_t);
4958             }
4959 #endif
4960         }
4961     }
4962     if (state == ARM_CP_STATE_AA64) {
4963         /* To allow abbreviation of ARMCPRegInfo
4964          * definitions, we treat cp == 0 as equivalent to
4965          * the value for "standard guest-visible sysreg".
4966          * STATE_BOTH definitions are also always "standard
4967          * sysreg" in their AArch64 view (the .cp value may
4968          * be non-zero for the benefit of the AArch32 view).
4969          */
4970         if (r->cp == 0 || r->state == ARM_CP_STATE_BOTH) {
4971             r2->cp = CP_REG_ARM64_SYSREG_CP;
4972         }
4973         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
4974                                   r2->opc0, opc1, opc2);
4975     } else {
4976         *key = ENCODE_CP_REG(r2->cp, is64, ns, r2->crn, crm, opc1, opc2);
4977     }
4978     if (opaque) {
4979         r2->opaque = opaque;
4980     }
4981     /* reginfo passed to helpers is correct for the actual access,
4982      * and is never ARM_CP_STATE_BOTH:
4983      */
4984     r2->state = state;
4985     /* Make sure reginfo passed to helpers for wildcarded regs
4986      * has the correct crm/opc1/opc2 for this reg, not CP_ANY:
4987      */
4988     r2->crm = crm;
4989     r2->opc1 = opc1;
4990     r2->opc2 = opc2;
4991     /* By convention, for wildcarded registers only the first
4992      * entry is used for migration; the others are marked as
4993      * ALIAS so we don't try to transfer the register
4994      * multiple times. Special registers (ie NOP/WFI) are
4995      * never migratable and not even raw-accessible.
4996      */
4997     if ((r->type & ARM_CP_SPECIAL)) {
4998         r2->type |= ARM_CP_NO_RAW;
4999     }
5000     if (((r->crm == CP_ANY) && crm != 0) ||
5001         ((r->opc1 == CP_ANY) && opc1 != 0) ||
5002         ((r->opc2 == CP_ANY) && opc2 != 0)) {
5003         r2->type |= ARM_CP_ALIAS;
5004     }
5005
5006     /* Check that raw accesses are either forbidden or handled. Note that
5007      * we can't assert this earlier because the setup of fieldoffset for
5008      * banked registers has to be done first.
5009      */
5010     if (!(r2->type & ARM_CP_NO_RAW)) {
5011         assert(!raw_accessors_invalid(r2));
5012     }
5013
5014     /* Overriding of an existing definition must be explicitly
5015      * requested.
5016      */
5017     if (!(r->type & ARM_CP_OVERRIDE)) {
5018         ARMCPRegInfo *oldreg;
5019         oldreg = g_hash_table_lookup(cpu->cp_regs, key);
5020         if (oldreg && !(oldreg->type & ARM_CP_OVERRIDE)) {
5021             fprintf(stderr, "Register redefined: cp=%d %d bit "
5022                     "crn=%d crm=%d opc1=%d opc2=%d, "
5023                     "was %s, now %s\n", r2->cp, 32 + 32 * is64,
5024                     r2->crn, r2->crm, r2->opc1, r2->opc2,
5025                     oldreg->name, r2->name);
5026             g_assert_not_reached();
5027         }
5028     }
5029     g_hash_table_insert(cpu->cp_regs, key, r2);
5030 }
5031
5032
5033 void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
5034                                        const ARMCPRegInfo *r, void *opaque)
5035 {
5036     /* Define implementations of coprocessor registers.
5037      * We store these in a hashtable because typically
5038      * there are less than 150 registers in a space which
5039      * is 16*16*16*8*8 = 262144 in size.
5040      * Wildcarding is supported for the crm, opc1 and opc2 fields.
5041      * If a register is defined twice then the second definition is
5042      * used, so this can be used to define some generic registers and
5043      * then override them with implementation specific variations.
5044      * At least one of the original and the second definition should
5045      * include ARM_CP_OVERRIDE in its type bits -- this is just a guard
5046      * against accidental use.
5047      *
5048      * The state field defines whether the register is to be
5049      * visible in the AArch32 or AArch64 execution state. If the
5050      * state is set to ARM_CP_STATE_BOTH then we synthesise a
5051      * reginfo structure for the AArch32 view, which sees the lower
5052      * 32 bits of the 64 bit register.
5053      *
5054      * Only registers visible in AArch64 may set r->opc0; opc0 cannot
5055      * be wildcarded. AArch64 registers are always considered to be 64
5056      * bits; the ARM_CP_64BIT* flag applies only to the AArch32 view of
5057      * the register, if any.
5058      */
5059     int crm, opc1, opc2, state;
5060     int crmmin = (r->crm == CP_ANY) ? 0 : r->crm;
5061     int crmmax = (r->crm == CP_ANY) ? 15 : r->crm;
5062     int opc1min = (r->opc1 == CP_ANY) ? 0 : r->opc1;
5063     int opc1max = (r->opc1 == CP_ANY) ? 7 : r->opc1;
5064     int opc2min = (r->opc2 == CP_ANY) ? 0 : r->opc2;
5065     int opc2max = (r->opc2 == CP_ANY) ? 7 : r->opc2;
5066     /* 64 bit registers have only CRm and Opc1 fields */
5067     assert(!((r->type & ARM_CP_64BIT) && (r->opc2 || r->crn)));
5068     /* op0 only exists in the AArch64 encodings */
5069     assert((r->state != ARM_CP_STATE_AA32) || (r->opc0 == 0));
5070     /* AArch64 regs are all 64 bit so ARM_CP_64BIT is meaningless */
5071     assert((r->state != ARM_CP_STATE_AA64) || !(r->type & ARM_CP_64BIT));
5072     /* The AArch64 pseudocode CheckSystemAccess() specifies that op1
5073      * encodes a minimum access level for the register. We roll this
5074      * runtime check into our general permission check code, so check
5075      * here that the reginfo's specified permissions are strict enough
5076      * to encompass the generic architectural permission check.
5077      */
5078     if (r->state != ARM_CP_STATE_AA32) {
5079         int mask = 0;
5080         switch (r->opc1) {
5081         case 0: case 1: case 2:
5082             /* min_EL EL1 */
5083             mask = PL1_RW;
5084             break;
5085         case 3:
5086             /* min_EL EL0 */
5087             mask = PL0_RW;
5088             break;
5089         case 4:
5090             /* min_EL EL2 */
5091             mask = PL2_RW;
5092             break;
5093         case 5:
5094             /* unallocated encoding, so not possible */
5095             assert(false);
5096             break;
5097         case 6:
5098             /* min_EL EL3 */
5099             mask = PL3_RW;
5100             break;
5101         case 7:
5102             /* min_EL EL1, secure mode only (we don't check the latter) */
5103             mask = PL1_RW;
5104             break;
5105         default:
5106             /* broken reginfo with out-of-range opc1 */
5107             assert(false);
5108             break;
5109         }
5110         /* assert our permissions are not too lax (stricter is fine) */
5111         assert((r->access & ~mask) == 0);
5112     }
5113
5114     /* Check that the register definition has enough info to handle
5115      * reads and writes if they are permitted.
5116      */
5117     if (!(r->type & (ARM_CP_SPECIAL|ARM_CP_CONST))) {
5118         if (r->access & PL3_R) {
5119             assert((r->fieldoffset ||
5120                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
5121                    r->readfn);
5122         }
5123         if (r->access & PL3_W) {
5124             assert((r->fieldoffset ||
5125                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
5126                    r->writefn);
5127         }
5128     }
5129     /* Bad type field probably means missing sentinel at end of reg list */
5130     assert(cptype_valid(r->type));
5131     for (crm = crmmin; crm <= crmmax; crm++) {
5132         for (opc1 = opc1min; opc1 <= opc1max; opc1++) {
5133             for (opc2 = opc2min; opc2 <= opc2max; opc2++) {
5134                 for (state = ARM_CP_STATE_AA32;
5135                      state <= ARM_CP_STATE_AA64; state++) {
5136                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
5137                         continue;
5138                     }
5139                     if (state == ARM_CP_STATE_AA32) {
5140                         /* Under AArch32 CP registers can be common
5141                          * (same for secure and non-secure world) or banked.
5142                          */
5143                         switch (r->secure) {
5144                         case ARM_CP_SECSTATE_S:
5145                         case ARM_CP_SECSTATE_NS:
5146                             add_cpreg_to_hashtable(cpu, r, opaque, state,
5147                                                    r->secure, crm, opc1, opc2);
5148                             break;
5149                         default:
5150                             add_cpreg_to_hashtable(cpu, r, opaque, state,
5151                                                    ARM_CP_SECSTATE_S,
5152                                                    crm, opc1, opc2);
5153                             add_cpreg_to_hashtable(cpu, r, opaque, state,
5154                                                    ARM_CP_SECSTATE_NS,
5155                                                    crm, opc1, opc2);
5156                             break;
5157                         }
5158                     } else {
5159                         /* AArch64 registers get mapped to non-secure instance
5160                          * of AArch32 */
5161                         add_cpreg_to_hashtable(cpu, r, opaque, state,
5162                                                ARM_CP_SECSTATE_NS,
5163                                                crm, opc1, opc2);
5164                     }
5165                 }
5166             }
5167         }
5168     }
5169 }
5170
5171 void define_arm_cp_regs_with_opaque(ARMCPU *cpu,
5172                                     const ARMCPRegInfo *regs, void *opaque)
5173 {
5174     /* Define a whole list of registers */
5175     const ARMCPRegInfo *r;
5176     for (r = regs; r->type != ARM_CP_SENTINEL; r++) {
5177         define_one_arm_cp_reg_with_opaque(cpu, r, opaque);
5178     }
5179 }
5180
5181 const ARMCPRegInfo *get_arm_cp_reginfo(GHashTable *cpregs, uint32_t encoded_cp)
5182 {
5183     return g_hash_table_lookup(cpregs, &encoded_cp);
5184 }
5185
5186 void arm_cp_write_ignore(CPUARMState *env, const ARMCPRegInfo *ri,
5187                          uint64_t value)
5188 {
5189     /* Helper coprocessor write function for write-ignore registers */
5190 }
5191
5192 uint64_t arm_cp_read_zero(CPUARMState *env, const ARMCPRegInfo *ri)
5193 {
5194     /* Helper coprocessor write function for read-as-zero registers */
5195     return 0;
5196 }
5197
5198 void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
5199 {
5200     /* Helper coprocessor reset function for do-nothing-on-reset registers */
5201 }
5202
5203 static int bad_mode_switch(CPUARMState *env, int mode)
5204 {
5205     /* Return true if it is not valid for us to switch to
5206      * this CPU mode (ie all the UNPREDICTABLE cases in
5207      * the ARM ARM CPSRWriteByInstr pseudocode).
5208      */
5209     switch (mode) {
5210     case ARM_CPU_MODE_USR:
5211     case ARM_CPU_MODE_SYS:
5212     case ARM_CPU_MODE_SVC:
5213     case ARM_CPU_MODE_ABT:
5214     case ARM_CPU_MODE_UND:
5215     case ARM_CPU_MODE_IRQ:
5216     case ARM_CPU_MODE_FIQ:
5217         return 0;
5218     case ARM_CPU_MODE_MON:
5219         return !arm_is_secure(env);
5220     default:
5221         return 1;
5222     }
5223 }
5224
5225 uint32_t cpsr_read(CPUARMState *env)
5226 {
5227     int ZF;
5228     ZF = (env->ZF == 0);
5229     return env->uncached_cpsr | (env->NF & 0x80000000) | (ZF << 30) |
5230         (env->CF << 29) | ((env->VF & 0x80000000) >> 3) | (env->QF << 27)
5231         | (env->thumb << 5) | ((env->condexec_bits & 3) << 25)
5232         | ((env->condexec_bits & 0xfc) << 8)
5233         | (env->GE << 16) | (env->daif & CPSR_AIF);
5234 }
5235
5236 void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
5237 {
5238     uint32_t changed_daif;
5239
5240     if (mask & CPSR_NZCV) {
5241         env->ZF = (~val) & CPSR_Z;
5242         env->NF = val;
5243         env->CF = (val >> 29) & 1;
5244         env->VF = (val << 3) & 0x80000000;
5245     }
5246     if (mask & CPSR_Q)
5247         env->QF = ((val & CPSR_Q) != 0);
5248     if (mask & CPSR_T)
5249         env->thumb = ((val & CPSR_T) != 0);
5250     if (mask & CPSR_IT_0_1) {
5251         env->condexec_bits &= ~3;
5252         env->condexec_bits |= (val >> 25) & 3;
5253     }
5254     if (mask & CPSR_IT_2_7) {
5255         env->condexec_bits &= 3;
5256         env->condexec_bits |= (val >> 8) & 0xfc;
5257     }
5258     if (mask & CPSR_GE) {
5259         env->GE = (val >> 16) & 0xf;
5260     }
5261
5262     /* In a V7 implementation that includes the security extensions but does
5263      * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
5264      * whether non-secure software is allowed to change the CPSR_F and CPSR_A
5265      * bits respectively.
5266      *
5267      * In a V8 implementation, it is permitted for privileged software to
5268      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
5269      */
5270     if (!arm_feature(env, ARM_FEATURE_V8) &&
5271         arm_feature(env, ARM_FEATURE_EL3) &&
5272         !arm_feature(env, ARM_FEATURE_EL2) &&
5273         !arm_is_secure(env)) {
5274
5275         changed_daif = (env->daif ^ val) & mask;
5276
5277         if (changed_daif & CPSR_A) {
5278             /* Check to see if we are allowed to change the masking of async
5279              * abort exceptions from a non-secure state.
5280              */
5281             if (!(env->cp15.scr_el3 & SCR_AW)) {
5282                 qemu_log_mask(LOG_GUEST_ERROR,
5283                               "Ignoring attempt to switch CPSR_A flag from "
5284                               "non-secure world with SCR.AW bit clear\n");
5285                 mask &= ~CPSR_A;
5286             }
5287         }
5288
5289         if (changed_daif & CPSR_F) {
5290             /* Check to see if we are allowed to change the masking of FIQ
5291              * exceptions from a non-secure state.
5292              */
5293             if (!(env->cp15.scr_el3 & SCR_FW)) {
5294                 qemu_log_mask(LOG_GUEST_ERROR,
5295                               "Ignoring attempt to switch CPSR_F flag from "
5296                               "non-secure world with SCR.FW bit clear\n");
5297                 mask &= ~CPSR_F;
5298             }
5299
5300             /* Check whether non-maskable FIQ (NMFI) support is enabled.
5301              * If this bit is set software is not allowed to mask
5302              * FIQs, but is allowed to set CPSR_F to 0.
5303              */
5304             if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
5305                 (val & CPSR_F)) {
5306                 qemu_log_mask(LOG_GUEST_ERROR,
5307                               "Ignoring attempt to enable CPSR_F flag "
5308                               "(non-maskable FIQ [NMFI] support enabled)\n");
5309                 mask &= ~CPSR_F;
5310             }
5311         }
5312     }
5313
5314     env->daif &= ~(CPSR_AIF & mask);
5315     env->daif |= val & CPSR_AIF & mask;
5316
5317     if ((env->uncached_cpsr ^ val) & mask & CPSR_M) {
5318         if (bad_mode_switch(env, val & CPSR_M)) {
5319             /* Attempt to switch to an invalid mode: this is UNPREDICTABLE.
5320              * We choose to ignore the attempt and leave the CPSR M field
5321              * untouched.
5322              */
5323             mask &= ~CPSR_M;
5324         } else {
5325             switch_mode(env, val & CPSR_M);
5326         }
5327     }
5328     mask &= ~CACHED_CPSR_BITS;
5329     env->uncached_cpsr = (env->uncached_cpsr & ~mask) | (val & mask);
5330 }
5331
5332 /* Sign/zero extend */
5333 uint32_t HELPER(sxtb16)(uint32_t x)
5334 {
5335     uint32_t res;
5336     res = (uint16_t)(int8_t)x;
5337     res |= (uint32_t)(int8_t)(x >> 16) << 16;
5338     return res;
5339 }
5340
5341 uint32_t HELPER(uxtb16)(uint32_t x)
5342 {
5343     uint32_t res;
5344     res = (uint16_t)(uint8_t)x;
5345     res |= (uint32_t)(uint8_t)(x >> 16) << 16;
5346     return res;
5347 }
5348
5349 uint32_t HELPER(clz)(uint32_t x)
5350 {
5351     return clz32(x);
5352 }
5353
5354 int32_t HELPER(sdiv)(int32_t num, int32_t den)
5355 {
5356     if (den == 0)
5357       return 0;
5358     if (num == INT_MIN && den == -1)
5359       return INT_MIN;
5360     return num / den;
5361 }
5362
5363 uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
5364 {
5365     if (den == 0)
5366       return 0;
5367     return num / den;
5368 }
5369
5370 uint32_t HELPER(rbit)(uint32_t x)
5371 {
5372     return revbit32(x);
5373 }
5374
5375 #if defined(CONFIG_USER_ONLY)
5376
5377 /* These should probably raise undefined insn exceptions.  */
5378 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
5379 {
5380     ARMCPU *cpu = arm_env_get_cpu(env);
5381
5382     cpu_abort(CPU(cpu), "v7m_msr %d\n", reg);
5383 }
5384
5385 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
5386 {
5387     ARMCPU *cpu = arm_env_get_cpu(env);
5388
5389     cpu_abort(CPU(cpu), "v7m_mrs %d\n", reg);
5390     return 0;
5391 }
5392
5393 void switch_mode(CPUARMState *env, int mode)
5394 {
5395     ARMCPU *cpu = arm_env_get_cpu(env);
5396
5397     if (mode != ARM_CPU_MODE_USR) {
5398         cpu_abort(CPU(cpu), "Tried to switch out of user mode\n");
5399     }
5400 }
5401
5402 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
5403                                  uint32_t cur_el, bool secure)
5404 {
5405     return 1;
5406 }
5407
5408 void aarch64_sync_64_to_32(CPUARMState *env)
5409 {
5410     g_assert_not_reached();
5411 }
5412
5413 #else
5414
5415 void switch_mode(CPUARMState *env, int mode)
5416 {
5417     int old_mode;
5418     int i;
5419
5420     old_mode = env->uncached_cpsr & CPSR_M;
5421     if (mode == old_mode)
5422         return;
5423
5424     if (old_mode == ARM_CPU_MODE_FIQ) {
5425         memcpy (env->fiq_regs, env->regs + 8, 5 * sizeof(uint32_t));
5426         memcpy (env->regs + 8, env->usr_regs, 5 * sizeof(uint32_t));
5427     } else if (mode == ARM_CPU_MODE_FIQ) {
5428         memcpy (env->usr_regs, env->regs + 8, 5 * sizeof(uint32_t));
5429         memcpy (env->regs + 8, env->fiq_regs, 5 * sizeof(uint32_t));
5430     }
5431
5432     i = bank_number(old_mode);
5433     env->banked_r13[i] = env->regs[13];
5434     env->banked_r14[i] = env->regs[14];
5435     env->banked_spsr[i] = env->spsr;
5436
5437     i = bank_number(mode);
5438     env->regs[13] = env->banked_r13[i];
5439     env->regs[14] = env->banked_r14[i];
5440     env->spsr = env->banked_spsr[i];
5441 }
5442
5443 /* Physical Interrupt Target EL Lookup Table
5444  *
5445  * [ From ARM ARM section G1.13.4 (Table G1-15) ]
5446  *
5447  * The below multi-dimensional table is used for looking up the target
5448  * exception level given numerous condition criteria.  Specifically, the
5449  * target EL is based on SCR and HCR routing controls as well as the
5450  * currently executing EL and secure state.
5451  *
5452  *    Dimensions:
5453  *    target_el_table[2][2][2][2][2][4]
5454  *                    |  |  |  |  |  +--- Current EL
5455  *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
5456  *                    |  |  |  +--------- HCR mask override
5457  *                    |  |  +------------ SCR exec state control
5458  *                    |  +--------------- SCR mask override
5459  *                    +------------------ 32-bit(0)/64-bit(1) EL3
5460  *
5461  *    The table values are as such:
5462  *    0-3 = EL0-EL3
5463  *     -1 = Cannot occur
5464  *
5465  * The ARM ARM target EL table includes entries indicating that an "exception
5466  * is not taken".  The two cases where this is applicable are:
5467  *    1) An exception is taken from EL3 but the SCR does not have the exception
5468  *    routed to EL3.
5469  *    2) An exception is taken from EL2 but the HCR does not have the exception
5470  *    routed to EL2.
5471  * In these two cases, the below table contain a target of EL1.  This value is
5472  * returned as it is expected that the consumer of the table data will check
5473  * for "target EL >= current EL" to ensure the exception is not taken.
5474  *
5475  *            SCR     HCR
5476  *         64  EA     AMO                 From
5477  *        BIT IRQ     IMO      Non-secure         Secure
5478  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
5479  */
5480 static const int8_t target_el_table[2][2][2][2][2][4] = {
5481     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
5482        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
5483       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
5484        {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
5485      {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
5486        {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
5487       {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
5488        {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
5489     {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
5490        {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},
5491       {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1, -1,  1 },},
5492        {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},},
5493      {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
5494        {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
5495       {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
5496        {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},},},
5497 };
5498
5499 /*
5500  * Determine the target EL for physical exceptions
5501  */
5502 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
5503                                  uint32_t cur_el, bool secure)
5504 {
5505     CPUARMState *env = cs->env_ptr;
5506     int rw;
5507     int scr;
5508     int hcr;
5509     int target_el;
5510     /* Is the highest EL AArch64? */
5511     int is64 = arm_feature(env, ARM_FEATURE_AARCH64);
5512
5513     if (arm_feature(env, ARM_FEATURE_EL3)) {
5514         rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
5515     } else {
5516         /* Either EL2 is the highest EL (and so the EL2 register width
5517          * is given by is64); or there is no EL2 or EL3, in which case
5518          * the value of 'rw' does not affect the table lookup anyway.
5519          */
5520         rw = is64;
5521     }
5522
5523     switch (excp_idx) {
5524     case EXCP_IRQ:
5525         scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
5526         hcr = ((env->cp15.hcr_el2 & HCR_IMO) == HCR_IMO);
5527         break;
5528     case EXCP_FIQ:
5529         scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
5530         hcr = ((env->cp15.hcr_el2 & HCR_FMO) == HCR_FMO);
5531         break;
5532     default:
5533         scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
5534         hcr = ((env->cp15.hcr_el2 & HCR_AMO) == HCR_AMO);
5535         break;
5536     };
5537
5538     /* If HCR.TGE is set then HCR is treated as being 1 */
5539     hcr |= ((env->cp15.hcr_el2 & HCR_TGE) == HCR_TGE);
5540
5541     /* Perform a table-lookup for the target EL given the current state */
5542     target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
5543
5544     assert(target_el > 0);
5545
5546     return target_el;
5547 }
5548
5549 static void v7m_push(CPUARMState *env, uint32_t val)
5550 {
5551     CPUState *cs = CPU(arm_env_get_cpu(env));
5552
5553     env->regs[13] -= 4;
5554     stl_phys(cs->as, env->regs[13], val);
5555 }
5556
5557 static uint32_t v7m_pop(CPUARMState *env)
5558 {
5559     CPUState *cs = CPU(arm_env_get_cpu(env));
5560     uint32_t val;
5561
5562     val = ldl_phys(cs->as, env->regs[13]);
5563     env->regs[13] += 4;
5564     return val;
5565 }
5566
5567 /* Switch to V7M main or process stack pointer.  */
5568 static void switch_v7m_sp(CPUARMState *env, int process)
5569 {
5570     uint32_t tmp;
5571     if (env->v7m.current_sp != process) {
5572         tmp = env->v7m.other_sp;
5573         env->v7m.other_sp = env->regs[13];
5574         env->regs[13] = tmp;
5575         env->v7m.current_sp = process;
5576     }
5577 }
5578
5579 static void do_v7m_exception_exit(CPUARMState *env)
5580 {
5581     uint32_t type;
5582     uint32_t xpsr;
5583
5584     type = env->regs[15];
5585     if (env->v7m.exception != 0)
5586         armv7m_nvic_complete_irq(env->nvic, env->v7m.exception);
5587
5588     /* Switch to the target stack.  */
5589     switch_v7m_sp(env, (type & 4) != 0);
5590     /* Pop registers.  */
5591     env->regs[0] = v7m_pop(env);
5592     env->regs[1] = v7m_pop(env);
5593     env->regs[2] = v7m_pop(env);
5594     env->regs[3] = v7m_pop(env);
5595     env->regs[12] = v7m_pop(env);
5596     env->regs[14] = v7m_pop(env);
5597     env->regs[15] = v7m_pop(env);
5598     if (env->regs[15] & 1) {
5599         qemu_log_mask(LOG_GUEST_ERROR,
5600                       "M profile return from interrupt with misaligned "
5601                       "PC is UNPREDICTABLE\n");
5602         /* Actual hardware seems to ignore the lsbit, and there are several
5603          * RTOSes out there which incorrectly assume the r15 in the stack
5604          * frame should be a Thumb-style "lsbit indicates ARM/Thumb" value.
5605          */
5606         env->regs[15] &= ~1U;
5607     }
5608     xpsr = v7m_pop(env);
5609     xpsr_write(env, xpsr, 0xfffffdff);
5610     /* Undo stack alignment.  */
5611     if (xpsr & 0x200)
5612         env->regs[13] |= 4;
5613     /* ??? The exception return type specifies Thread/Handler mode.  However
5614        this is also implied by the xPSR value. Not sure what to do
5615        if there is a mismatch.  */
5616     /* ??? Likewise for mismatches between the CONTROL register and the stack
5617        pointer.  */
5618 }
5619
5620 void arm_v7m_cpu_do_interrupt(CPUState *cs)
5621 {
5622     ARMCPU *cpu = ARM_CPU(cs);
5623     CPUARMState *env = &cpu->env;
5624     uint32_t xpsr = xpsr_read(env);
5625     uint32_t lr;
5626     uint32_t addr;
5627
5628     arm_log_exception(cs->exception_index);
5629
5630     lr = 0xfffffff1;
5631     if (env->v7m.current_sp)
5632         lr |= 4;
5633     if (env->v7m.exception == 0)
5634         lr |= 8;
5635
5636     /* For exceptions we just mark as pending on the NVIC, and let that
5637        handle it.  */
5638     /* TODO: Need to escalate if the current priority is higher than the
5639        one we're raising.  */
5640     switch (cs->exception_index) {
5641     case EXCP_UDEF:
5642         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE);
5643         return;
5644     case EXCP_SWI:
5645         /* The PC already points to the next instruction.  */
5646         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SVC);
5647         return;
5648     case EXCP_PREFETCH_ABORT:
5649     case EXCP_DATA_ABORT:
5650         /* TODO: if we implemented the MPU registers, this is where we
5651          * should set the MMFAR, etc from exception.fsr and exception.vaddress.
5652          */
5653         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_MEM);
5654         return;
5655     case EXCP_BKPT:
5656         if (semihosting_enabled()) {
5657             int nr;
5658             nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
5659             if (nr == 0xab) {
5660                 env->regs[15] += 2;
5661                 qemu_log_mask(CPU_LOG_INT,
5662                               "...handling as semihosting call 0x%x\n",
5663                               env->regs[0]);
5664                 env->regs[0] = do_arm_semihosting(env);
5665                 return;
5666             }
5667         }
5668         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_DEBUG);
5669         return;
5670     case EXCP_IRQ:
5671         env->v7m.exception = armv7m_nvic_acknowledge_irq(env->nvic);
5672         break;
5673     case EXCP_EXCEPTION_EXIT:
5674         do_v7m_exception_exit(env);
5675         return;
5676     default:
5677         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
5678         return; /* Never happens.  Keep compiler happy.  */
5679     }
5680
5681     /* Align stack pointer.  */
5682     /* ??? Should only do this if Configuration Control Register
5683        STACKALIGN bit is set.  */
5684     if (env->regs[13] & 4) {
5685         env->regs[13] -= 4;
5686         xpsr |= 0x200;
5687     }
5688     /* Switch to the handler mode.  */
5689     v7m_push(env, xpsr);
5690     v7m_push(env, env->regs[15]);
5691     v7m_push(env, env->regs[14]);
5692     v7m_push(env, env->regs[12]);
5693     v7m_push(env, env->regs[3]);
5694     v7m_push(env, env->regs[2]);
5695     v7m_push(env, env->regs[1]);
5696     v7m_push(env, env->regs[0]);
5697     switch_v7m_sp(env, 0);
5698     /* Clear IT bits */
5699     env->condexec_bits = 0;
5700     env->regs[14] = lr;
5701     addr = ldl_phys(cs->as, env->v7m.vecbase + env->v7m.exception * 4);
5702     env->regs[15] = addr & 0xfffffffe;
5703     env->thumb = addr & 1;
5704 }
5705
5706 /* Function used to synchronize QEMU's AArch64 register set with AArch32
5707  * register set.  This is necessary when switching between AArch32 and AArch64
5708  * execution state.
5709  */
5710 void aarch64_sync_32_to_64(CPUARMState *env)
5711 {
5712     int i;
5713     uint32_t mode = env->uncached_cpsr & CPSR_M;
5714
5715     /* We can blanket copy R[0:7] to X[0:7] */
5716     for (i = 0; i < 8; i++) {
5717         env->xregs[i] = env->regs[i];
5718     }
5719
5720     /* Unless we are in FIQ mode, x8-x12 come from the user registers r8-r12.
5721      * Otherwise, they come from the banked user regs.
5722      */
5723     if (mode == ARM_CPU_MODE_FIQ) {
5724         for (i = 8; i < 13; i++) {
5725             env->xregs[i] = env->usr_regs[i - 8];
5726         }
5727     } else {
5728         for (i = 8; i < 13; i++) {
5729             env->xregs[i] = env->regs[i];
5730         }
5731     }
5732
5733     /* Registers x13-x23 are the various mode SP and FP registers. Registers
5734      * r13 and r14 are only copied if we are in that mode, otherwise we copy
5735      * from the mode banked register.
5736      */
5737     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
5738         env->xregs[13] = env->regs[13];
5739         env->xregs[14] = env->regs[14];
5740     } else {
5741         env->xregs[13] = env->banked_r13[bank_number(ARM_CPU_MODE_USR)];
5742         /* HYP is an exception in that it is copied from r14 */
5743         if (mode == ARM_CPU_MODE_HYP) {
5744             env->xregs[14] = env->regs[14];
5745         } else {
5746             env->xregs[14] = env->banked_r14[bank_number(ARM_CPU_MODE_USR)];
5747         }
5748     }
5749
5750     if (mode == ARM_CPU_MODE_HYP) {
5751         env->xregs[15] = env->regs[13];
5752     } else {
5753         env->xregs[15] = env->banked_r13[bank_number(ARM_CPU_MODE_HYP)];
5754     }
5755
5756     if (mode == ARM_CPU_MODE_IRQ) {
5757         env->xregs[16] = env->regs[14];
5758         env->xregs[17] = env->regs[13];
5759     } else {
5760         env->xregs[16] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
5761         env->xregs[17] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
5762     }
5763
5764     if (mode == ARM_CPU_MODE_SVC) {
5765         env->xregs[18] = env->regs[14];
5766         env->xregs[19] = env->regs[13];
5767     } else {
5768         env->xregs[18] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
5769         env->xregs[19] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
5770     }
5771
5772     if (mode == ARM_CPU_MODE_ABT) {
5773         env->xregs[20] = env->regs[14];
5774         env->xregs[21] = env->regs[13];
5775     } else {
5776         env->xregs[20] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
5777         env->xregs[21] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
5778     }
5779
5780     if (mode == ARM_CPU_MODE_UND) {
5781         env->xregs[22] = env->regs[14];
5782         env->xregs[23] = env->regs[13];
5783     } else {
5784         env->xregs[22] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
5785         env->xregs[23] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
5786     }
5787
5788     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
5789      * mode, then we can copy from r8-r14.  Otherwise, we copy from the
5790      * FIQ bank for r8-r14.
5791      */
5792     if (mode == ARM_CPU_MODE_FIQ) {
5793         for (i = 24; i < 31; i++) {
5794             env->xregs[i] = env->regs[i - 16];   /* X[24:30] <- R[8:14] */
5795         }
5796     } else {
5797         for (i = 24; i < 29; i++) {
5798             env->xregs[i] = env->fiq_regs[i - 24];
5799         }
5800         env->xregs[29] = env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)];
5801         env->xregs[30] = env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)];
5802     }
5803
5804     env->pc = env->regs[15];
5805 }
5806
5807 /* Function used to synchronize QEMU's AArch32 register set with AArch64
5808  * register set.  This is necessary when switching between AArch32 and AArch64
5809  * execution state.
5810  */
5811 void aarch64_sync_64_to_32(CPUARMState *env)
5812 {
5813     int i;
5814     uint32_t mode = env->uncached_cpsr & CPSR_M;
5815
5816     /* We can blanket copy X[0:7] to R[0:7] */
5817     for (i = 0; i < 8; i++) {
5818         env->regs[i] = env->xregs[i];
5819     }
5820
5821     /* Unless we are in FIQ mode, r8-r12 come from the user registers x8-x12.
5822      * Otherwise, we copy x8-x12 into the banked user regs.
5823      */
5824     if (mode == ARM_CPU_MODE_FIQ) {
5825         for (i = 8; i < 13; i++) {
5826             env->usr_regs[i - 8] = env->xregs[i];
5827         }
5828     } else {
5829         for (i = 8; i < 13; i++) {
5830             env->regs[i] = env->xregs[i];
5831         }
5832     }
5833
5834     /* Registers r13 & r14 depend on the current mode.
5835      * If we are in a given mode, we copy the corresponding x registers to r13
5836      * and r14.  Otherwise, we copy the x register to the banked r13 and r14
5837      * for the mode.
5838      */
5839     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
5840         env->regs[13] = env->xregs[13];
5841         env->regs[14] = env->xregs[14];
5842     } else {
5843         env->banked_r13[bank_number(ARM_CPU_MODE_USR)] = env->xregs[13];
5844
5845         /* HYP is an exception in that it does not have its own banked r14 but
5846          * shares the USR r14
5847          */
5848         if (mode == ARM_CPU_MODE_HYP) {
5849             env->regs[14] = env->xregs[14];
5850         } else {
5851             env->banked_r14[bank_number(ARM_CPU_MODE_USR)] = env->xregs[14];
5852         }
5853     }
5854
5855     if (mode == ARM_CPU_MODE_HYP) {
5856         env->regs[13] = env->xregs[15];
5857     } else {
5858         env->banked_r13[bank_number(ARM_CPU_MODE_HYP)] = env->xregs[15];
5859     }
5860
5861     if (mode == ARM_CPU_MODE_IRQ) {
5862         env->regs[14] = env->xregs[16];
5863         env->regs[13] = env->xregs[17];
5864     } else {
5865         env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
5866         env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
5867     }
5868
5869     if (mode == ARM_CPU_MODE_SVC) {
5870         env->regs[14] = env->xregs[18];
5871         env->regs[13] = env->xregs[19];
5872     } else {
5873         env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
5874         env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
5875     }
5876
5877     if (mode == ARM_CPU_MODE_ABT) {
5878         env->regs[14] = env->xregs[20];
5879         env->regs[13] = env->xregs[21];
5880     } else {
5881         env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
5882         env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
5883     }
5884
5885     if (mode == ARM_CPU_MODE_UND) {
5886         env->regs[14] = env->xregs[22];
5887         env->regs[13] = env->xregs[23];
5888     } else {
5889         env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
5890         env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
5891     }
5892
5893     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
5894      * mode, then we can copy to r8-r14.  Otherwise, we copy to the
5895      * FIQ bank for r8-r14.
5896      */
5897     if (mode == ARM_CPU_MODE_FIQ) {
5898         for (i = 24; i < 31; i++) {
5899             env->regs[i - 16] = env->xregs[i];   /* X[24:30] -> R[8:14] */
5900         }
5901     } else {
5902         for (i = 24; i < 29; i++) {
5903             env->fiq_regs[i - 24] = env->xregs[i];
5904         }
5905         env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[29];
5906         env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[30];
5907     }
5908
5909     env->regs[15] = env->pc;
5910 }
5911
5912 static void arm_cpu_do_interrupt_aarch32(CPUState *cs)
5913 {
5914     ARMCPU *cpu = ARM_CPU(cs);
5915     CPUARMState *env = &cpu->env;
5916     uint32_t addr;
5917     uint32_t mask;
5918     int new_mode;
5919     uint32_t offset;
5920     uint32_t moe;
5921
5922     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
5923     switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
5924     case EC_BREAKPOINT:
5925     case EC_BREAKPOINT_SAME_EL:
5926         moe = 1;
5927         break;
5928     case EC_WATCHPOINT:
5929     case EC_WATCHPOINT_SAME_EL:
5930         moe = 10;
5931         break;
5932     case EC_AA32_BKPT:
5933         moe = 3;
5934         break;
5935     case EC_VECTORCATCH:
5936         moe = 5;
5937         break;
5938     default:
5939         moe = 0;
5940         break;
5941     }
5942
5943     if (moe) {
5944         env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
5945     }
5946
5947     /* TODO: Vectored interrupt controller.  */
5948     switch (cs->exception_index) {
5949     case EXCP_UDEF:
5950         new_mode = ARM_CPU_MODE_UND;
5951         addr = 0x04;
5952         mask = CPSR_I;
5953         if (env->thumb)
5954             offset = 2;
5955         else
5956             offset = 4;
5957         break;
5958     case EXCP_SWI:
5959         new_mode = ARM_CPU_MODE_SVC;
5960         addr = 0x08;
5961         mask = CPSR_I;
5962         /* The PC already points to the next instruction.  */
5963         offset = 0;
5964         break;
5965     case EXCP_BKPT:
5966         env->exception.fsr = 2;
5967         /* Fall through to prefetch abort.  */
5968     case EXCP_PREFETCH_ABORT:
5969         A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
5970         A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
5971         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
5972                       env->exception.fsr, (uint32_t)env->exception.vaddress);
5973         new_mode = ARM_CPU_MODE_ABT;
5974         addr = 0x0c;
5975         mask = CPSR_A | CPSR_I;
5976         offset = 4;
5977         break;
5978     case EXCP_DATA_ABORT:
5979         A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
5980         A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
5981         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
5982                       env->exception.fsr,
5983                       (uint32_t)env->exception.vaddress);
5984         new_mode = ARM_CPU_MODE_ABT;
5985         addr = 0x10;
5986         mask = CPSR_A | CPSR_I;
5987         offset = 8;
5988         break;
5989     case EXCP_IRQ:
5990         new_mode = ARM_CPU_MODE_IRQ;
5991         addr = 0x18;
5992         /* Disable IRQ and imprecise data aborts.  */
5993         mask = CPSR_A | CPSR_I;
5994         offset = 4;
5995         if (env->cp15.scr_el3 & SCR_IRQ) {
5996             /* IRQ routed to monitor mode */
5997             new_mode = ARM_CPU_MODE_MON;
5998             mask |= CPSR_F;
5999         }
6000         break;
6001     case EXCP_FIQ:
6002         new_mode = ARM_CPU_MODE_FIQ;
6003         addr = 0x1c;
6004         /* Disable FIQ, IRQ and imprecise data aborts.  */
6005         mask = CPSR_A | CPSR_I | CPSR_F;
6006         if (env->cp15.scr_el3 & SCR_FIQ) {
6007             /* FIQ routed to monitor mode */
6008             new_mode = ARM_CPU_MODE_MON;
6009         }
6010         offset = 4;
6011         break;
6012     case EXCP_SMC:
6013         new_mode = ARM_CPU_MODE_MON;
6014         addr = 0x08;
6015         mask = CPSR_A | CPSR_I | CPSR_F;
6016         offset = 0;
6017         break;
6018     default:
6019         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
6020         return; /* Never happens.  Keep compiler happy.  */
6021     }
6022
6023     if (new_mode == ARM_CPU_MODE_MON) {
6024         addr += env->cp15.mvbar;
6025     } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
6026         /* High vectors. When enabled, base address cannot be remapped. */
6027         addr += 0xffff0000;
6028     } else {
6029         /* ARM v7 architectures provide a vector base address register to remap
6030          * the interrupt vector table.
6031          * This register is only followed in non-monitor mode, and is banked.
6032          * Note: only bits 31:5 are valid.
6033          */
6034         addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
6035     }
6036
6037     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
6038         env->cp15.scr_el3 &= ~SCR_NS;
6039     }
6040
6041     switch_mode (env, new_mode);
6042     /* For exceptions taken to AArch32 we must clear the SS bit in both
6043      * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
6044      */
6045     env->uncached_cpsr &= ~PSTATE_SS;
6046     env->spsr = cpsr_read(env);
6047     /* Clear IT bits.  */
6048     env->condexec_bits = 0;
6049     /* Switch to the new mode, and to the correct instruction set.  */
6050     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
6051     env->daif |= mask;
6052     /* this is a lie, as the was no c1_sys on V4T/V5, but who cares
6053      * and we should just guard the thumb mode on V4 */
6054     if (arm_feature(env, ARM_FEATURE_V4T)) {
6055         env->thumb = (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
6056     }
6057     env->regs[14] = env->regs[15] + offset;
6058     env->regs[15] = addr;
6059 }
6060
6061 /* Handle exception entry to a target EL which is using AArch64 */
6062 static void arm_cpu_do_interrupt_aarch64(CPUState *cs)
6063 {
6064     ARMCPU *cpu = ARM_CPU(cs);
6065     CPUARMState *env = &cpu->env;
6066     unsigned int new_el = env->exception.target_el;
6067     target_ulong addr = env->cp15.vbar_el[new_el];
6068     unsigned int new_mode = aarch64_pstate_mode(new_el, true);
6069
6070     if (arm_current_el(env) < new_el) {
6071         /* Entry vector offset depends on whether the implemented EL
6072          * immediately lower than the target level is using AArch32 or AArch64
6073          */
6074         bool is_aa64;
6075
6076         switch (new_el) {
6077         case 3:
6078             is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0;
6079             break;
6080         case 2:
6081             is_aa64 = (env->cp15.hcr_el2 & HCR_RW) != 0;
6082             break;
6083         case 1:
6084             is_aa64 = is_a64(env);
6085             break;
6086         default:
6087             g_assert_not_reached();
6088         }
6089
6090         if (is_aa64) {
6091             addr += 0x400;
6092         } else {
6093             addr += 0x600;
6094         }
6095     } else if (pstate_read(env) & PSTATE_SP) {
6096         addr += 0x200;
6097     }
6098
6099     switch (cs->exception_index) {
6100     case EXCP_PREFETCH_ABORT:
6101     case EXCP_DATA_ABORT:
6102         env->cp15.far_el[new_el] = env->exception.vaddress;
6103         qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
6104                       env->cp15.far_el[new_el]);
6105         /* fall through */
6106     case EXCP_BKPT:
6107     case EXCP_UDEF:
6108     case EXCP_SWI:
6109     case EXCP_HVC:
6110     case EXCP_HYP_TRAP:
6111     case EXCP_SMC:
6112         env->cp15.esr_el[new_el] = env->exception.syndrome;
6113         break;
6114     case EXCP_IRQ:
6115     case EXCP_VIRQ:
6116         addr += 0x80;
6117         break;
6118     case EXCP_FIQ:
6119     case EXCP_VFIQ:
6120         addr += 0x100;
6121         break;
6122     case EXCP_SEMIHOST:
6123         qemu_log_mask(CPU_LOG_INT,
6124                       "...handling as semihosting call 0x%" PRIx64 "\n",
6125                       env->xregs[0]);
6126         env->xregs[0] = do_arm_semihosting(env);
6127         return;
6128     default:
6129         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
6130     }
6131
6132     if (is_a64(env)) {
6133         env->banked_spsr[aarch64_banked_spsr_index(new_el)] = pstate_read(env);
6134         aarch64_save_sp(env, arm_current_el(env));
6135         env->elr_el[new_el] = env->pc;
6136     } else {
6137         env->banked_spsr[aarch64_banked_spsr_index(new_el)] = cpsr_read(env);
6138         if (!env->thumb) {
6139             env->cp15.esr_el[new_el] |= 1 << 25;
6140         }
6141         env->elr_el[new_el] = env->regs[15];
6142
6143         aarch64_sync_32_to_64(env);
6144
6145         env->condexec_bits = 0;
6146     }
6147     qemu_log_mask(CPU_LOG_INT, "...with ELR 0x%" PRIx64 "\n",
6148                   env->elr_el[new_el]);
6149
6150     pstate_write(env, PSTATE_DAIF | new_mode);
6151     env->aarch64 = 1;
6152     aarch64_restore_sp(env, new_el);
6153
6154     env->pc = addr;
6155
6156     qemu_log_mask(CPU_LOG_INT, "...to EL%d PC 0x%" PRIx64 " PSTATE 0x%x\n",
6157                   new_el, env->pc, pstate_read(env));
6158 }
6159
6160 static inline bool check_for_semihosting(CPUState *cs)
6161 {
6162     /* Check whether this exception is a semihosting call; if so
6163      * then handle it and return true; otherwise return false.
6164      */
6165     ARMCPU *cpu = ARM_CPU(cs);
6166     CPUARMState *env = &cpu->env;
6167
6168     if (is_a64(env)) {
6169         if (cs->exception_index == EXCP_SEMIHOST) {
6170             /* This is always the 64-bit semihosting exception.
6171              * The "is this usermode" and "is semihosting enabled"
6172              * checks have been done at translate time.
6173              */
6174             qemu_log_mask(CPU_LOG_INT,
6175                           "...handling as semihosting call 0x%" PRIx64 "\n",
6176                           env->xregs[0]);
6177             env->xregs[0] = do_arm_semihosting(env);
6178             return true;
6179         }
6180         return false;
6181     } else {
6182         uint32_t imm;
6183
6184         /* Only intercept calls from privileged modes, to provide some
6185          * semblance of security.
6186          */
6187         if (!semihosting_enabled() ||
6188             ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)) {
6189             return false;
6190         }
6191
6192         switch (cs->exception_index) {
6193         case EXCP_SWI:
6194             /* Check for semihosting interrupt.  */
6195             if (env->thumb) {
6196                 imm = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
6197                     & 0xff;
6198                 if (imm == 0xab) {
6199                     break;
6200                 }
6201             } else {
6202                 imm = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
6203                     & 0xffffff;
6204                 if (imm == 0x123456) {
6205                     break;
6206                 }
6207             }
6208             return false;
6209         case EXCP_BKPT:
6210             /* See if this is a semihosting syscall.  */
6211             if (env->thumb) {
6212                 imm = arm_lduw_code(env, env->regs[15], env->bswap_code)
6213                     & 0xff;
6214                 if (imm == 0xab) {
6215                     env->regs[15] += 2;
6216                     break;
6217                 }
6218             }
6219             return false;
6220         default:
6221             return false;
6222         }
6223
6224         qemu_log_mask(CPU_LOG_INT,
6225                       "...handling as semihosting call 0x%x\n",
6226                       env->regs[0]);
6227         env->regs[0] = do_arm_semihosting(env);
6228         return true;
6229     }
6230 }
6231
6232 /* Handle a CPU exception for A and R profile CPUs.
6233  * Do any appropriate logging, handle PSCI calls, and then hand off
6234  * to the AArch64-entry or AArch32-entry function depending on the
6235  * target exception level's register width.
6236  */
6237 void arm_cpu_do_interrupt(CPUState *cs)
6238 {
6239     ARMCPU *cpu = ARM_CPU(cs);
6240     CPUARMState *env = &cpu->env;
6241     unsigned int new_el = env->exception.target_el;
6242
6243     assert(!IS_M(env));
6244
6245     arm_log_exception(cs->exception_index);
6246     qemu_log_mask(CPU_LOG_INT, "...from EL%d to EL%d\n", arm_current_el(env),
6247                   new_el);
6248     if (qemu_loglevel_mask(CPU_LOG_INT)
6249         && !excp_is_internal(cs->exception_index)) {
6250         qemu_log_mask(CPU_LOG_INT, "...with ESR %x/0x%" PRIx32 "\n",
6251                       env->exception.syndrome >> ARM_EL_EC_SHIFT,
6252                       env->exception.syndrome);
6253     }
6254
6255     if (arm_is_psci_call(cpu, cs->exception_index)) {
6256         arm_handle_psci_call(cpu);
6257         qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
6258         return;
6259     }
6260
6261     /* Semihosting semantics depend on the register width of the
6262      * code that caused the exception, not the target exception level,
6263      * so must be handled here.
6264      */
6265     if (check_for_semihosting(cs)) {
6266         return;
6267     }
6268
6269     assert(!excp_is_internal(cs->exception_index));
6270     if (arm_el_is_aa64(env, new_el)) {
6271         arm_cpu_do_interrupt_aarch64(cs);
6272     } else {
6273         arm_cpu_do_interrupt_aarch32(cs);
6274     }
6275
6276     if (!kvm_enabled()) {
6277         cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
6278     }
6279 }
6280
6281 /* Return the exception level which controls this address translation regime */
6282 static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx)
6283 {
6284     switch (mmu_idx) {
6285     case ARMMMUIdx_S2NS:
6286     case ARMMMUIdx_S1E2:
6287         return 2;
6288     case ARMMMUIdx_S1E3:
6289         return 3;
6290     case ARMMMUIdx_S1SE0:
6291         return arm_el_is_aa64(env, 3) ? 1 : 3;
6292     case ARMMMUIdx_S1SE1:
6293     case ARMMMUIdx_S1NSE0:
6294     case ARMMMUIdx_S1NSE1:
6295         return 1;
6296     default:
6297         g_assert_not_reached();
6298     }
6299 }
6300
6301 /* Return true if this address translation regime is secure */
6302 static inline bool regime_is_secure(CPUARMState *env, ARMMMUIdx mmu_idx)
6303 {
6304     switch (mmu_idx) {
6305     case ARMMMUIdx_S12NSE0:
6306     case ARMMMUIdx_S12NSE1:
6307     case ARMMMUIdx_S1NSE0:
6308     case ARMMMUIdx_S1NSE1:
6309     case ARMMMUIdx_S1E2:
6310     case ARMMMUIdx_S2NS:
6311         return false;
6312     case ARMMMUIdx_S1E3:
6313     case ARMMMUIdx_S1SE0:
6314     case ARMMMUIdx_S1SE1:
6315         return true;
6316     default:
6317         g_assert_not_reached();
6318     }
6319 }
6320
6321 /* Return the SCTLR value which controls this address translation regime */
6322 static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx)
6323 {
6324     return env->cp15.sctlr_el[regime_el(env, mmu_idx)];
6325 }
6326
6327 /* Return true if the specified stage of address translation is disabled */
6328 static inline bool regime_translation_disabled(CPUARMState *env,
6329                                                ARMMMUIdx mmu_idx)
6330 {
6331     if (mmu_idx == ARMMMUIdx_S2NS) {
6332         return (env->cp15.hcr_el2 & HCR_VM) == 0;
6333     }
6334     return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
6335 }
6336
6337 /* Return the TCR controlling this translation regime */
6338 static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx)
6339 {
6340     if (mmu_idx == ARMMMUIdx_S2NS) {
6341         return &env->cp15.vtcr_el2;
6342     }
6343     return &env->cp15.tcr_el[regime_el(env, mmu_idx)];
6344 }
6345
6346 /* Return the TTBR associated with this translation regime */
6347 static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx,
6348                                    int ttbrn)
6349 {
6350     if (mmu_idx == ARMMMUIdx_S2NS) {
6351         return env->cp15.vttbr_el2;
6352     }
6353     if (ttbrn == 0) {
6354         return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
6355     } else {
6356         return env->cp15.ttbr1_el[regime_el(env, mmu_idx)];
6357     }
6358 }
6359
6360 /* Return true if the translation regime is using LPAE format page tables */
6361 static inline bool regime_using_lpae_format(CPUARMState *env,
6362                                             ARMMMUIdx mmu_idx)
6363 {
6364     int el = regime_el(env, mmu_idx);
6365     if (el == 2 || arm_el_is_aa64(env, el)) {
6366         return true;
6367     }
6368     if (arm_feature(env, ARM_FEATURE_LPAE)
6369         && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) {
6370         return true;
6371     }
6372     return false;
6373 }
6374
6375 /* Returns true if the stage 1 translation regime is using LPAE format page
6376  * tables. Used when raising alignment exceptions, whose FSR changes depending
6377  * on whether the long or short descriptor format is in use. */
6378 bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx)
6379 {
6380     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
6381         mmu_idx += ARMMMUIdx_S1NSE0;
6382     }
6383
6384     return regime_using_lpae_format(env, mmu_idx);
6385 }
6386
6387 static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx)
6388 {
6389     switch (mmu_idx) {
6390     case ARMMMUIdx_S1SE0:
6391     case ARMMMUIdx_S1NSE0:
6392         return true;
6393     default:
6394         return false;
6395     case ARMMMUIdx_S12NSE0:
6396     case ARMMMUIdx_S12NSE1:
6397         g_assert_not_reached();
6398     }
6399 }
6400
6401 /* Translate section/page access permissions to page
6402  * R/W protection flags
6403  *
6404  * @env:         CPUARMState
6405  * @mmu_idx:     MMU index indicating required translation regime
6406  * @ap:          The 3-bit access permissions (AP[2:0])
6407  * @domain_prot: The 2-bit domain access permissions
6408  */
6409 static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
6410                                 int ap, int domain_prot)
6411 {
6412     bool is_user = regime_is_user(env, mmu_idx);
6413
6414     if (domain_prot == 3) {
6415         return PAGE_READ | PAGE_WRITE;
6416     }
6417
6418     switch (ap) {
6419     case 0:
6420         if (arm_feature(env, ARM_FEATURE_V7)) {
6421             return 0;
6422         }
6423         switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) {
6424         case SCTLR_S:
6425             return is_user ? 0 : PAGE_READ;
6426         case SCTLR_R:
6427             return PAGE_READ;
6428         default:
6429             return 0;
6430         }
6431     case 1:
6432         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
6433     case 2:
6434         if (is_user) {
6435             return PAGE_READ;
6436         } else {
6437             return PAGE_READ | PAGE_WRITE;
6438         }
6439     case 3:
6440         return PAGE_READ | PAGE_WRITE;
6441     case 4: /* Reserved.  */
6442         return 0;
6443     case 5:
6444         return is_user ? 0 : PAGE_READ;
6445     case 6:
6446         return PAGE_READ;
6447     case 7:
6448         if (!arm_feature(env, ARM_FEATURE_V6K)) {
6449             return 0;
6450         }
6451         return PAGE_READ;
6452     default:
6453         g_assert_not_reached();
6454     }
6455 }
6456
6457 /* Translate section/page access permissions to page
6458  * R/W protection flags.
6459  *
6460  * @ap:      The 2-bit simple AP (AP[2:1])
6461  * @is_user: TRUE if accessing from PL0
6462  */
6463 static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
6464 {
6465     switch (ap) {
6466     case 0:
6467         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
6468     case 1:
6469         return PAGE_READ | PAGE_WRITE;
6470     case 2:
6471         return is_user ? 0 : PAGE_READ;
6472     case 3:
6473         return PAGE_READ;
6474     default:
6475         g_assert_not_reached();
6476     }
6477 }
6478
6479 static inline int
6480 simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
6481 {
6482     return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
6483 }
6484
6485 /* Translate S2 section/page access permissions to protection flags
6486  *
6487  * @env:     CPUARMState
6488  * @s2ap:    The 2-bit stage2 access permissions (S2AP)
6489  * @xn:      XN (execute-never) bit
6490  */
6491 static int get_S2prot(CPUARMState *env, int s2ap, int xn)
6492 {
6493     int prot = 0;
6494
6495     if (s2ap & 1) {
6496         prot |= PAGE_READ;
6497     }
6498     if (s2ap & 2) {
6499         prot |= PAGE_WRITE;
6500     }
6501     if (!xn) {
6502         prot |= PAGE_EXEC;
6503     }
6504     return prot;
6505 }
6506
6507 /* Translate section/page access permissions to protection flags
6508  *
6509  * @env:     CPUARMState
6510  * @mmu_idx: MMU index indicating required translation regime
6511  * @is_aa64: TRUE if AArch64
6512  * @ap:      The 2-bit simple AP (AP[2:1])
6513  * @ns:      NS (non-secure) bit
6514  * @xn:      XN (execute-never) bit
6515  * @pxn:     PXN (privileged execute-never) bit
6516  */
6517 static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
6518                       int ap, int ns, int xn, int pxn)
6519 {
6520     bool is_user = regime_is_user(env, mmu_idx);
6521     int prot_rw, user_rw;
6522     bool have_wxn;
6523     int wxn = 0;
6524
6525     assert(mmu_idx != ARMMMUIdx_S2NS);
6526
6527     user_rw = simple_ap_to_rw_prot_is_user(ap, true);
6528     if (is_user) {
6529         prot_rw = user_rw;
6530     } else {
6531         prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
6532     }
6533
6534     if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) {
6535         return prot_rw;
6536     }
6537
6538     /* TODO have_wxn should be replaced with
6539      *   ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
6540      * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
6541      * compatible processors have EL2, which is required for [U]WXN.
6542      */
6543     have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
6544
6545     if (have_wxn) {
6546         wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
6547     }
6548
6549     if (is_aa64) {
6550         switch (regime_el(env, mmu_idx)) {
6551         case 1:
6552             if (!is_user) {
6553                 xn = pxn || (user_rw & PAGE_WRITE);
6554             }
6555             break;
6556         case 2:
6557         case 3:
6558             break;
6559         }
6560     } else if (arm_feature(env, ARM_FEATURE_V7)) {
6561         switch (regime_el(env, mmu_idx)) {
6562         case 1:
6563         case 3:
6564             if (is_user) {
6565                 xn = xn || !(user_rw & PAGE_READ);
6566             } else {
6567                 int uwxn = 0;
6568                 if (have_wxn) {
6569                     uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
6570                 }
6571                 xn = xn || !(prot_rw & PAGE_READ) || pxn ||
6572                      (uwxn && (user_rw & PAGE_WRITE));
6573             }
6574             break;
6575         case 2:
6576             break;
6577         }
6578     } else {
6579         xn = wxn = 0;
6580     }
6581
6582     if (xn || (wxn && (prot_rw & PAGE_WRITE))) {
6583         return prot_rw;
6584     }
6585     return prot_rw | PAGE_EXEC;
6586 }
6587
6588 static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
6589                                      uint32_t *table, uint32_t address)
6590 {
6591     /* Note that we can only get here for an AArch32 PL0/PL1 lookup */
6592     TCR *tcr = regime_tcr(env, mmu_idx);
6593
6594     if (address & tcr->mask) {
6595         if (tcr->raw_tcr & TTBCR_PD1) {
6596             /* Translation table walk disabled for TTBR1 */
6597             return false;
6598         }
6599         *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000;
6600     } else {
6601         if (tcr->raw_tcr & TTBCR_PD0) {
6602             /* Translation table walk disabled for TTBR0 */
6603             return false;
6604         }
6605         *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask;
6606     }
6607     *table |= (address >> 18) & 0x3ffc;
6608     return true;
6609 }
6610
6611 /* Translate a S1 pagetable walk through S2 if needed.  */
6612 static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx,
6613                                hwaddr addr, MemTxAttrs txattrs,
6614                                uint32_t *fsr,
6615                                ARMMMUFaultInfo *fi)
6616 {
6617     if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) &&
6618         !regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
6619         target_ulong s2size;
6620         hwaddr s2pa;
6621         int s2prot;
6622         int ret;
6623
6624         ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa,
6625                                  &txattrs, &s2prot, &s2size, fsr, fi);
6626         if (ret) {
6627             fi->s2addr = addr;
6628             fi->stage2 = true;
6629             fi->s1ptw = true;
6630             return ~0;
6631         }
6632         addr = s2pa;
6633     }
6634     return addr;
6635 }
6636
6637 /* All loads done in the course of a page table walk go through here.
6638  * TODO: rather than ignoring errors from physical memory reads (which
6639  * are external aborts in ARM terminology) we should propagate this
6640  * error out so that we can turn it into a Data Abort if this walk
6641  * was being done for a CPU load/store or an address translation instruction
6642  * (but not if it was for a debug access).
6643  */
6644 static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure,
6645                             ARMMMUIdx mmu_idx, uint32_t *fsr,
6646                             ARMMMUFaultInfo *fi)
6647 {
6648     ARMCPU *cpu = ARM_CPU(cs);
6649     CPUARMState *env = &cpu->env;
6650     MemTxAttrs attrs = {};
6651     AddressSpace *as;
6652
6653     attrs.secure = is_secure;
6654     as = arm_addressspace(cs, attrs);
6655     addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
6656     if (fi->s1ptw) {
6657         return 0;
6658     }
6659     return address_space_ldl(as, addr, attrs, NULL);
6660 }
6661
6662 static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure,
6663                             ARMMMUIdx mmu_idx, uint32_t *fsr,
6664                             ARMMMUFaultInfo *fi)
6665 {
6666     ARMCPU *cpu = ARM_CPU(cs);
6667     CPUARMState *env = &cpu->env;
6668     MemTxAttrs attrs = {};
6669     AddressSpace *as;
6670
6671     attrs.secure = is_secure;
6672     as = arm_addressspace(cs, attrs);
6673     addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
6674     if (fi->s1ptw) {
6675         return 0;
6676     }
6677     return address_space_ldq(as, addr, attrs, NULL);
6678 }
6679
6680 static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
6681                              int access_type, ARMMMUIdx mmu_idx,
6682                              hwaddr *phys_ptr, int *prot,
6683                              target_ulong *page_size, uint32_t *fsr,
6684                              ARMMMUFaultInfo *fi)
6685 {
6686     CPUState *cs = CPU(arm_env_get_cpu(env));
6687     int code;
6688     uint32_t table;
6689     uint32_t desc;
6690     int type;
6691     int ap;
6692     int domain = 0;
6693     int domain_prot;
6694     hwaddr phys_addr;
6695     uint32_t dacr;
6696
6697     /* Pagetable walk.  */
6698     /* Lookup l1 descriptor.  */
6699     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
6700         /* Section translation fault if page walk is disabled by PD0 or PD1 */
6701         code = 5;
6702         goto do_fault;
6703     }
6704     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
6705                        mmu_idx, fsr, fi);
6706     type = (desc & 3);
6707     domain = (desc >> 5) & 0x0f;
6708     if (regime_el(env, mmu_idx) == 1) {
6709         dacr = env->cp15.dacr_ns;
6710     } else {
6711         dacr = env->cp15.dacr_s;
6712     }
6713     domain_prot = (dacr >> (domain * 2)) & 3;
6714     if (type == 0) {
6715         /* Section translation fault.  */
6716         code = 5;
6717         goto do_fault;
6718     }
6719     if (domain_prot == 0 || domain_prot == 2) {
6720         if (type == 2)
6721             code = 9; /* Section domain fault.  */
6722         else
6723             code = 11; /* Page domain fault.  */
6724         goto do_fault;
6725     }
6726     if (type == 2) {
6727         /* 1Mb section.  */
6728         phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
6729         ap = (desc >> 10) & 3;
6730         code = 13;
6731         *page_size = 1024 * 1024;
6732     } else {
6733         /* Lookup l2 entry.  */
6734         if (type == 1) {
6735             /* Coarse pagetable.  */
6736             table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
6737         } else {
6738             /* Fine pagetable.  */
6739             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
6740         }
6741         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
6742                            mmu_idx, fsr, fi);
6743         switch (desc & 3) {
6744         case 0: /* Page translation fault.  */
6745             code = 7;
6746             goto do_fault;
6747         case 1: /* 64k page.  */
6748             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
6749             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
6750             *page_size = 0x10000;
6751             break;
6752         case 2: /* 4k page.  */
6753             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
6754             ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
6755             *page_size = 0x1000;
6756             break;
6757         case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */
6758             if (type == 1) {
6759                 /* ARMv6/XScale extended small page format */
6760                 if (arm_feature(env, ARM_FEATURE_XSCALE)
6761                     || arm_feature(env, ARM_FEATURE_V6)) {
6762                     phys_addr = (desc & 0xfffff000) | (address & 0xfff);
6763                     *page_size = 0x1000;
6764                 } else {
6765                     /* UNPREDICTABLE in ARMv5; we choose to take a
6766                      * page translation fault.
6767                      */
6768                     code = 7;
6769                     goto do_fault;
6770                 }
6771             } else {
6772                 phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
6773                 *page_size = 0x400;
6774             }
6775             ap = (desc >> 4) & 3;
6776             break;
6777         default:
6778             /* Never happens, but compiler isn't smart enough to tell.  */
6779             abort();
6780         }
6781         code = 15;
6782     }
6783     *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
6784     *prot |= *prot ? PAGE_EXEC : 0;
6785     if (!(*prot & (1 << access_type))) {
6786         /* Access permission fault.  */
6787         goto do_fault;
6788     }
6789     *phys_ptr = phys_addr;
6790     return false;
6791 do_fault:
6792     *fsr = code | (domain << 4);
6793     return true;
6794 }
6795
6796 static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
6797                              int access_type, ARMMMUIdx mmu_idx,
6798                              hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
6799                              target_ulong *page_size, uint32_t *fsr,
6800                              ARMMMUFaultInfo *fi)
6801 {
6802     CPUState *cs = CPU(arm_env_get_cpu(env));
6803     int code;
6804     uint32_t table;
6805     uint32_t desc;
6806     uint32_t xn;
6807     uint32_t pxn = 0;
6808     int type;
6809     int ap;
6810     int domain = 0;
6811     int domain_prot;
6812     hwaddr phys_addr;
6813     uint32_t dacr;
6814     bool ns;
6815
6816     /* Pagetable walk.  */
6817     /* Lookup l1 descriptor.  */
6818     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
6819         /* Section translation fault if page walk is disabled by PD0 or PD1 */
6820         code = 5;
6821         goto do_fault;
6822     }
6823     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
6824                        mmu_idx, fsr, fi);
6825     type = (desc & 3);
6826     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
6827         /* Section translation fault, or attempt to use the encoding
6828          * which is Reserved on implementations without PXN.
6829          */
6830         code = 5;
6831         goto do_fault;
6832     }
6833     if ((type == 1) || !(desc & (1 << 18))) {
6834         /* Page or Section.  */
6835         domain = (desc >> 5) & 0x0f;
6836     }
6837     if (regime_el(env, mmu_idx) == 1) {
6838         dacr = env->cp15.dacr_ns;
6839     } else {
6840         dacr = env->cp15.dacr_s;
6841     }
6842     domain_prot = (dacr >> (domain * 2)) & 3;
6843     if (domain_prot == 0 || domain_prot == 2) {
6844         if (type != 1) {
6845             code = 9; /* Section domain fault.  */
6846         } else {
6847             code = 11; /* Page domain fault.  */
6848         }
6849         goto do_fault;
6850     }
6851     if (type != 1) {
6852         if (desc & (1 << 18)) {
6853             /* Supersection.  */
6854             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
6855             phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32;
6856             phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36;
6857             *page_size = 0x1000000;
6858         } else {
6859             /* Section.  */
6860             phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
6861             *page_size = 0x100000;
6862         }
6863         ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
6864         xn = desc & (1 << 4);
6865         pxn = desc & 1;
6866         code = 13;
6867         ns = extract32(desc, 19, 1);
6868     } else {
6869         if (arm_feature(env, ARM_FEATURE_PXN)) {
6870             pxn = (desc >> 2) & 1;
6871         }
6872         ns = extract32(desc, 3, 1);
6873         /* Lookup l2 entry.  */
6874         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
6875         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
6876                            mmu_idx, fsr, fi);
6877         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
6878         switch (desc & 3) {
6879         case 0: /* Page translation fault.  */
6880             code = 7;
6881             goto do_fault;
6882         case 1: /* 64k page.  */
6883             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
6884             xn = desc & (1 << 15);
6885             *page_size = 0x10000;
6886             break;
6887         case 2: case 3: /* 4k page.  */
6888             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
6889             xn = desc & 1;
6890             *page_size = 0x1000;
6891             break;
6892         default:
6893             /* Never happens, but compiler isn't smart enough to tell.  */
6894             abort();
6895         }
6896         code = 15;
6897     }
6898     if (domain_prot == 3) {
6899         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
6900     } else {
6901         if (pxn && !regime_is_user(env, mmu_idx)) {
6902             xn = 1;
6903         }
6904         if (xn && access_type == 2)
6905             goto do_fault;
6906
6907         if (arm_feature(env, ARM_FEATURE_V6K) &&
6908                 (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
6909             /* The simplified model uses AP[0] as an access control bit.  */
6910             if ((ap & 1) == 0) {
6911                 /* Access flag fault.  */
6912                 code = (code == 15) ? 6 : 3;
6913                 goto do_fault;
6914             }
6915             *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
6916         } else {
6917             *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
6918         }
6919         if (*prot && !xn) {
6920             *prot |= PAGE_EXEC;
6921         }
6922         if (!(*prot & (1 << access_type))) {
6923             /* Access permission fault.  */
6924             goto do_fault;
6925         }
6926     }
6927     if (ns) {
6928         /* The NS bit will (as required by the architecture) have no effect if
6929          * the CPU doesn't support TZ or this is a non-secure translation
6930          * regime, because the attribute will already be non-secure.
6931          */
6932         attrs->secure = false;
6933     }
6934     *phys_ptr = phys_addr;
6935     return false;
6936 do_fault:
6937     *fsr = code | (domain << 4);
6938     return true;
6939 }
6940
6941 /* Fault type for long-descriptor MMU fault reporting; this corresponds
6942  * to bits [5..2] in the STATUS field in long-format DFSR/IFSR.
6943  */
6944 typedef enum {
6945     translation_fault = 1,
6946     access_fault = 2,
6947     permission_fault = 3,
6948 } MMUFaultType;
6949
6950 /*
6951  * check_s2_mmu_setup
6952  * @cpu:        ARMCPU
6953  * @is_aa64:    True if the translation regime is in AArch64 state
6954  * @startlevel: Suggested starting level
6955  * @inputsize:  Bitsize of IPAs
6956  * @stride:     Page-table stride (See the ARM ARM)
6957  *
6958  * Returns true if the suggested S2 translation parameters are OK and
6959  * false otherwise.
6960  */
6961 static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level,
6962                                int inputsize, int stride)
6963 {
6964     const int grainsize = stride + 3;
6965     int startsizecheck;
6966
6967     /* Negative levels are never allowed.  */
6968     if (level < 0) {
6969         return false;
6970     }
6971
6972     startsizecheck = inputsize - ((3 - level) * stride + grainsize);
6973     if (startsizecheck < 1 || startsizecheck > stride + 4) {
6974         return false;
6975     }
6976
6977     if (is_aa64) {
6978         CPUARMState *env = &cpu->env;
6979         unsigned int pamax = arm_pamax(cpu);
6980
6981         switch (stride) {
6982         case 13: /* 64KB Pages.  */
6983             if (level == 0 || (level == 1 && pamax <= 42)) {
6984                 return false;
6985             }
6986             break;
6987         case 11: /* 16KB Pages.  */
6988             if (level == 0 || (level == 1 && pamax <= 40)) {
6989                 return false;
6990             }
6991             break;
6992         case 9: /* 4KB Pages.  */
6993             if (level == 0 && pamax <= 42) {
6994                 return false;
6995             }
6996             break;
6997         default:
6998             g_assert_not_reached();
6999         }
7000
7001         /* Inputsize checks.  */
7002         if (inputsize > pamax &&
7003             (arm_el_is_aa64(env, 1) || inputsize > 40)) {
7004             /* This is CONSTRAINED UNPREDICTABLE and we choose to fault.  */
7005             return false;
7006         }
7007     } else {
7008         /* AArch32 only supports 4KB pages. Assert on that.  */
7009         assert(stride == 9);
7010
7011         if (level == 0) {
7012             return false;
7013         }
7014     }
7015     return true;
7016 }
7017
7018 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
7019                                int access_type, ARMMMUIdx mmu_idx,
7020                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
7021                                target_ulong *page_size_ptr, uint32_t *fsr,
7022                                ARMMMUFaultInfo *fi)
7023 {
7024     ARMCPU *cpu = arm_env_get_cpu(env);
7025     CPUState *cs = CPU(cpu);
7026     /* Read an LPAE long-descriptor translation table. */
7027     MMUFaultType fault_type = translation_fault;
7028     uint32_t level = 1;
7029     uint32_t epd = 0;
7030     int32_t t0sz, t1sz;
7031     uint32_t tg;
7032     uint64_t ttbr;
7033     int ttbr_select;
7034     hwaddr descaddr, descmask;
7035     uint32_t tableattrs;
7036     target_ulong page_size;
7037     uint32_t attrs;
7038     int32_t stride = 9;
7039     int32_t va_size = 32;
7040     int inputsize;
7041     int32_t tbi = 0;
7042     TCR *tcr = regime_tcr(env, mmu_idx);
7043     int ap, ns, xn, pxn;
7044     uint32_t el = regime_el(env, mmu_idx);
7045     bool ttbr1_valid = true;
7046     uint64_t descaddrmask;
7047
7048     /* TODO:
7049      * This code does not handle the different format TCR for VTCR_EL2.
7050      * This code also does not support shareability levels.
7051      * Attribute and permission bit handling should also be checked when adding
7052      * support for those page table walks.
7053      */
7054     if (arm_el_is_aa64(env, el)) {
7055         va_size = 64;
7056         if (el > 1) {
7057             if (mmu_idx != ARMMMUIdx_S2NS) {
7058                 tbi = extract64(tcr->raw_tcr, 20, 1);
7059             }
7060         } else {
7061             if (extract64(address, 55, 1)) {
7062                 tbi = extract64(tcr->raw_tcr, 38, 1);
7063             } else {
7064                 tbi = extract64(tcr->raw_tcr, 37, 1);
7065             }
7066         }
7067         tbi *= 8;
7068
7069         /* If we are in 64-bit EL2 or EL3 then there is no TTBR1, so mark it
7070          * invalid.
7071          */
7072         if (el > 1) {
7073             ttbr1_valid = false;
7074         }
7075     } else {
7076         /* There is no TTBR1 for EL2 */
7077         if (el == 2) {
7078             ttbr1_valid = false;
7079         }
7080     }
7081
7082     /* Determine whether this address is in the region controlled by
7083      * TTBR0 or TTBR1 (or if it is in neither region and should fault).
7084      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
7085      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
7086      */
7087     if (va_size == 64) {
7088         /* AArch64 translation.  */
7089         t0sz = extract32(tcr->raw_tcr, 0, 6);
7090         t0sz = MIN(t0sz, 39);
7091         t0sz = MAX(t0sz, 16);
7092     } else if (mmu_idx != ARMMMUIdx_S2NS) {
7093         /* AArch32 stage 1 translation.  */
7094         t0sz = extract32(tcr->raw_tcr, 0, 3);
7095     } else {
7096         /* AArch32 stage 2 translation.  */
7097         bool sext = extract32(tcr->raw_tcr, 4, 1);
7098         bool sign = extract32(tcr->raw_tcr, 3, 1);
7099         t0sz = sextract32(tcr->raw_tcr, 0, 4);
7100
7101         /* If the sign-extend bit is not the same as t0sz[3], the result
7102          * is unpredictable. Flag this as a guest error.  */
7103         if (sign != sext) {
7104             qemu_log_mask(LOG_GUEST_ERROR,
7105                           "AArch32: VTCR.S / VTCR.T0SZ[3] missmatch\n");
7106         }
7107     }
7108     t1sz = extract32(tcr->raw_tcr, 16, 6);
7109     if (va_size == 64) {
7110         t1sz = MIN(t1sz, 39);
7111         t1sz = MAX(t1sz, 16);
7112     }
7113     if (t0sz && !extract64(address, va_size - t0sz, t0sz - tbi)) {
7114         /* there is a ttbr0 region and we are in it (high bits all zero) */
7115         ttbr_select = 0;
7116     } else if (ttbr1_valid && t1sz &&
7117                !extract64(~address, va_size - t1sz, t1sz - tbi)) {
7118         /* there is a ttbr1 region and we are in it (high bits all one) */
7119         ttbr_select = 1;
7120     } else if (!t0sz) {
7121         /* ttbr0 region is "everything not in the ttbr1 region" */
7122         ttbr_select = 0;
7123     } else if (!t1sz && ttbr1_valid) {
7124         /* ttbr1 region is "everything not in the ttbr0 region" */
7125         ttbr_select = 1;
7126     } else {
7127         /* in the gap between the two regions, this is a Translation fault */
7128         fault_type = translation_fault;
7129         goto do_fault;
7130     }
7131
7132     /* Note that QEMU ignores shareability and cacheability attributes,
7133      * so we don't need to do anything with the SH, ORGN, IRGN fields
7134      * in the TTBCR.  Similarly, TTBCR:A1 selects whether we get the
7135      * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
7136      * implement any ASID-like capability so we can ignore it (instead
7137      * we will always flush the TLB any time the ASID is changed).
7138      */
7139     if (ttbr_select == 0) {
7140         ttbr = regime_ttbr(env, mmu_idx, 0);
7141         if (el < 2) {
7142             epd = extract32(tcr->raw_tcr, 7, 1);
7143         }
7144         inputsize = va_size - t0sz;
7145
7146         tg = extract32(tcr->raw_tcr, 14, 2);
7147         if (tg == 1) { /* 64KB pages */
7148             stride = 13;
7149         }
7150         if (tg == 2) { /* 16KB pages */
7151             stride = 11;
7152         }
7153     } else {
7154         /* We should only be here if TTBR1 is valid */
7155         assert(ttbr1_valid);
7156
7157         ttbr = regime_ttbr(env, mmu_idx, 1);
7158         epd = extract32(tcr->raw_tcr, 23, 1);
7159         inputsize = va_size - t1sz;
7160
7161         tg = extract32(tcr->raw_tcr, 30, 2);
7162         if (tg == 3)  { /* 64KB pages */
7163             stride = 13;
7164         }
7165         if (tg == 1) { /* 16KB pages */
7166             stride = 11;
7167         }
7168     }
7169
7170     /* Here we should have set up all the parameters for the translation:
7171      * va_size, inputsize, ttbr, epd, stride, tbi
7172      */
7173
7174     if (epd) {
7175         /* Translation table walk disabled => Translation fault on TLB miss
7176          * Note: This is always 0 on 64-bit EL2 and EL3.
7177          */
7178         goto do_fault;
7179     }
7180
7181     if (mmu_idx != ARMMMUIdx_S2NS) {
7182         /* The starting level depends on the virtual address size (which can
7183          * be up to 48 bits) and the translation granule size. It indicates
7184          * the number of strides (stride bits at a time) needed to
7185          * consume the bits of the input address. In the pseudocode this is:
7186          *  level = 4 - RoundUp((inputsize - grainsize) / stride)
7187          * where their 'inputsize' is our 'inputsize', 'grainsize' is
7188          * our 'stride + 3' and 'stride' is our 'stride'.
7189          * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
7190          * = 4 - (inputsize - stride - 3 + stride - 1) / stride
7191          * = 4 - (inputsize - 4) / stride;
7192          */
7193         level = 4 - (inputsize - 4) / stride;
7194     } else {
7195         /* For stage 2 translations the starting level is specified by the
7196          * VTCR_EL2.SL0 field (whose interpretation depends on the page size)
7197          */
7198         int startlevel = extract32(tcr->raw_tcr, 6, 2);
7199         bool ok;
7200
7201         if (va_size == 32 || stride == 9) {
7202             /* AArch32 or 4KB pages */
7203             level = 2 - startlevel;
7204         } else {
7205             /* 16KB or 64KB pages */
7206             level = 3 - startlevel;
7207         }
7208
7209         /* Check that the starting level is valid. */
7210         ok = check_s2_mmu_setup(cpu, va_size == 64, level, inputsize, stride);
7211         if (!ok) {
7212             /* AArch64 reports these as level 0 faults.
7213              * AArch32 reports these as level 1 faults.
7214              */
7215             level = va_size == 64 ? 0 : 1;
7216             fault_type = translation_fault;
7217             goto do_fault;
7218         }
7219     }
7220
7221     /* Clear the vaddr bits which aren't part of the within-region address,
7222      * so that we don't have to special case things when calculating the
7223      * first descriptor address.
7224      */
7225     if (va_size != inputsize) {
7226         address &= (1ULL << inputsize) - 1;
7227     }
7228
7229     descmask = (1ULL << (stride + 3)) - 1;
7230
7231     /* Now we can extract the actual base address from the TTBR */
7232     descaddr = extract64(ttbr, 0, 48);
7233     descaddr &= ~((1ULL << (inputsize - (stride * (4 - level)))) - 1);
7234
7235     /* The address field in the descriptor goes up to bit 39 for ARMv7
7236      * but up to bit 47 for ARMv8.
7237      */
7238     if (arm_feature(env, ARM_FEATURE_V8)) {
7239         descaddrmask = 0xfffffffff000ULL;
7240     } else {
7241         descaddrmask = 0xfffffff000ULL;
7242     }
7243
7244     /* Secure accesses start with the page table in secure memory and
7245      * can be downgraded to non-secure at any step. Non-secure accesses
7246      * remain non-secure. We implement this by just ORing in the NSTable/NS
7247      * bits at each step.
7248      */
7249     tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4);
7250     for (;;) {
7251         uint64_t descriptor;
7252         bool nstable;
7253
7254         descaddr |= (address >> (stride * (4 - level))) & descmask;
7255         descaddr &= ~7ULL;
7256         nstable = extract32(tableattrs, 4, 1);
7257         descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fsr, fi);
7258         if (fi->s1ptw) {
7259             goto do_fault;
7260         }
7261
7262         if (!(descriptor & 1) ||
7263             (!(descriptor & 2) && (level == 3))) {
7264             /* Invalid, or the Reserved level 3 encoding */
7265             goto do_fault;
7266         }
7267         descaddr = descriptor & descaddrmask;
7268
7269         if ((descriptor & 2) && (level < 3)) {
7270             /* Table entry. The top five bits are attributes which  may
7271              * propagate down through lower levels of the table (and
7272              * which are all arranged so that 0 means "no effect", so
7273              * we can gather them up by ORing in the bits at each level).
7274              */
7275             tableattrs |= extract64(descriptor, 59, 5);
7276             level++;
7277             continue;
7278         }
7279         /* Block entry at level 1 or 2, or page entry at level 3.
7280          * These are basically the same thing, although the number
7281          * of bits we pull in from the vaddr varies.
7282          */
7283         page_size = (1ULL << ((stride * (4 - level)) + 3));
7284         descaddr |= (address & (page_size - 1));
7285         /* Extract attributes from the descriptor */
7286         attrs = extract64(descriptor, 2, 10)
7287             | (extract64(descriptor, 52, 12) << 10);
7288
7289         if (mmu_idx == ARMMMUIdx_S2NS) {
7290             /* Stage 2 table descriptors do not include any attribute fields */
7291             break;
7292         }
7293         /* Merge in attributes from table descriptors */
7294         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
7295         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
7296         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
7297          * means "force PL1 access only", which means forcing AP[1] to 0.
7298          */
7299         if (extract32(tableattrs, 2, 1)) {
7300             attrs &= ~(1 << 4);
7301         }
7302         attrs |= nstable << 3; /* NS */
7303         break;
7304     }
7305     /* Here descaddr is the final physical address, and attributes
7306      * are all in attrs.
7307      */
7308     fault_type = access_fault;
7309     if ((attrs & (1 << 8)) == 0) {
7310         /* Access flag */
7311         goto do_fault;
7312     }
7313
7314     ap = extract32(attrs, 4, 2);
7315     xn = extract32(attrs, 12, 1);
7316
7317     if (mmu_idx == ARMMMUIdx_S2NS) {
7318         ns = true;
7319         *prot = get_S2prot(env, ap, xn);
7320     } else {
7321         ns = extract32(attrs, 3, 1);
7322         pxn = extract32(attrs, 11, 1);
7323         *prot = get_S1prot(env, mmu_idx, va_size == 64, ap, ns, xn, pxn);
7324     }
7325
7326     fault_type = permission_fault;
7327     if (!(*prot & (1 << access_type))) {
7328         goto do_fault;
7329     }
7330
7331     if (ns) {
7332         /* The NS bit will (as required by the architecture) have no effect if
7333          * the CPU doesn't support TZ or this is a non-secure translation
7334          * regime, because the attribute will already be non-secure.
7335          */
7336         txattrs->secure = false;
7337     }
7338     *phys_ptr = descaddr;
7339     *page_size_ptr = page_size;
7340     return false;
7341
7342 do_fault:
7343     /* Long-descriptor format IFSR/DFSR value */
7344     *fsr = (1 << 9) | (fault_type << 2) | level;
7345     /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2.  */
7346     fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS);
7347     return true;
7348 }
7349
7350 static inline void get_phys_addr_pmsav7_default(CPUARMState *env,
7351                                                 ARMMMUIdx mmu_idx,
7352                                                 int32_t address, int *prot)
7353 {
7354     *prot = PAGE_READ | PAGE_WRITE;
7355     switch (address) {
7356     case 0xF0000000 ... 0xFFFFFFFF:
7357         if (regime_sctlr(env, mmu_idx) & SCTLR_V) { /* hivecs execing is ok */
7358             *prot |= PAGE_EXEC;
7359         }
7360         break;
7361     case 0x00000000 ... 0x7FFFFFFF:
7362         *prot |= PAGE_EXEC;
7363         break;
7364     }
7365
7366 }
7367
7368 static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address,
7369                                  int access_type, ARMMMUIdx mmu_idx,
7370                                  hwaddr *phys_ptr, int *prot, uint32_t *fsr)
7371 {
7372     ARMCPU *cpu = arm_env_get_cpu(env);
7373     int n;
7374     bool is_user = regime_is_user(env, mmu_idx);
7375
7376     *phys_ptr = address;
7377     *prot = 0;
7378
7379     if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */
7380         get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
7381     } else { /* MPU enabled */
7382         for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) {
7383             /* region search */
7384             uint32_t base = env->pmsav7.drbar[n];
7385             uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5);
7386             uint32_t rmask;
7387             bool srdis = false;
7388
7389             if (!(env->pmsav7.drsr[n] & 0x1)) {
7390                 continue;
7391             }
7392
7393             if (!rsize) {
7394                 qemu_log_mask(LOG_GUEST_ERROR, "DRSR.Rsize field can not be 0");
7395                 continue;
7396             }
7397             rsize++;
7398             rmask = (1ull << rsize) - 1;
7399
7400             if (base & rmask) {
7401                 qemu_log_mask(LOG_GUEST_ERROR, "DRBAR %" PRIx32 " misaligned "
7402                               "to DRSR region size, mask = %" PRIx32,
7403                               base, rmask);
7404                 continue;
7405             }
7406
7407             if (address < base || address > base + rmask) {
7408                 continue;
7409             }
7410
7411             /* Region matched */
7412
7413             if (rsize >= 8) { /* no subregions for regions < 256 bytes */
7414                 int i, snd;
7415                 uint32_t srdis_mask;
7416
7417                 rsize -= 3; /* sub region size (power of 2) */
7418                 snd = ((address - base) >> rsize) & 0x7;
7419                 srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1);
7420
7421                 srdis_mask = srdis ? 0x3 : 0x0;
7422                 for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) {
7423                     /* This will check in groups of 2, 4 and then 8, whether
7424                      * the subregion bits are consistent. rsize is incremented
7425                      * back up to give the region size, considering consistent
7426                      * adjacent subregions as one region. Stop testing if rsize
7427                      * is already big enough for an entire QEMU page.
7428                      */
7429                     int snd_rounded = snd & ~(i - 1);
7430                     uint32_t srdis_multi = extract32(env->pmsav7.drsr[n],
7431                                                      snd_rounded + 8, i);
7432                     if (srdis_mask ^ srdis_multi) {
7433                         break;
7434                     }
7435                     srdis_mask = (srdis_mask << i) | srdis_mask;
7436                     rsize++;
7437                 }
7438             }
7439             if (rsize < TARGET_PAGE_BITS) {
7440                 qemu_log_mask(LOG_UNIMP, "No support for MPU (sub)region"
7441                               "alignment of %" PRIu32 " bits. Minimum is %d\n",
7442                               rsize, TARGET_PAGE_BITS);
7443                 continue;
7444             }
7445             if (srdis) {
7446                 continue;
7447             }
7448             break;
7449         }
7450
7451         if (n == -1) { /* no hits */
7452             if (cpu->pmsav7_dregion &&
7453                 (is_user || !(regime_sctlr(env, mmu_idx) & SCTLR_BR))) {
7454                 /* background fault */
7455                 *fsr = 0;
7456                 return true;
7457             }
7458             get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
7459         } else { /* a MPU hit! */
7460             uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3);
7461
7462             if (is_user) { /* User mode AP bit decoding */
7463                 switch (ap) {
7464                 case 0:
7465                 case 1:
7466                 case 5:
7467                     break; /* no access */
7468                 case 3:
7469                     *prot |= PAGE_WRITE;
7470                     /* fall through */
7471                 case 2:
7472                 case 6:
7473                     *prot |= PAGE_READ | PAGE_EXEC;
7474                     break;
7475                 default:
7476                     qemu_log_mask(LOG_GUEST_ERROR,
7477                                   "Bad value for AP bits in DRACR %"
7478                                   PRIx32 "\n", ap);
7479                 }
7480             } else { /* Priv. mode AP bits decoding */
7481                 switch (ap) {
7482                 case 0:
7483                     break; /* no access */
7484                 case 1:
7485                 case 2:
7486                 case 3:
7487                     *prot |= PAGE_WRITE;
7488                     /* fall through */
7489                 case 5:
7490                 case 6:
7491                     *prot |= PAGE_READ | PAGE_EXEC;
7492                     break;
7493                 default:
7494                     qemu_log_mask(LOG_GUEST_ERROR,
7495                                   "Bad value for AP bits in DRACR %"
7496                                   PRIx32 "\n", ap);
7497                 }
7498             }
7499
7500             /* execute never */
7501             if (env->pmsav7.dracr[n] & (1 << 12)) {
7502                 *prot &= ~PAGE_EXEC;
7503             }
7504         }
7505     }
7506
7507     *fsr = 0x00d; /* Permission fault */
7508     return !(*prot & (1 << access_type));
7509 }
7510
7511 static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address,
7512                                  int access_type, ARMMMUIdx mmu_idx,
7513                                  hwaddr *phys_ptr, int *prot, uint32_t *fsr)
7514 {
7515     int n;
7516     uint32_t mask;
7517     uint32_t base;
7518     bool is_user = regime_is_user(env, mmu_idx);
7519
7520     *phys_ptr = address;
7521     for (n = 7; n >= 0; n--) {
7522         base = env->cp15.c6_region[n];
7523         if ((base & 1) == 0) {
7524             continue;
7525         }
7526         mask = 1 << ((base >> 1) & 0x1f);
7527         /* Keep this shift separate from the above to avoid an
7528            (undefined) << 32.  */
7529         mask = (mask << 1) - 1;
7530         if (((base ^ address) & ~mask) == 0) {
7531             break;
7532         }
7533     }
7534     if (n < 0) {
7535         *fsr = 2;
7536         return true;
7537     }
7538
7539     if (access_type == 2) {
7540         mask = env->cp15.pmsav5_insn_ap;
7541     } else {
7542         mask = env->cp15.pmsav5_data_ap;
7543     }
7544     mask = (mask >> (n * 4)) & 0xf;
7545     switch (mask) {
7546     case 0:
7547         *fsr = 1;
7548         return true;
7549     case 1:
7550         if (is_user) {
7551             *fsr = 1;
7552             return true;
7553         }
7554         *prot = PAGE_READ | PAGE_WRITE;
7555         break;
7556     case 2:
7557         *prot = PAGE_READ;
7558         if (!is_user) {
7559             *prot |= PAGE_WRITE;
7560         }
7561         break;
7562     case 3:
7563         *prot = PAGE_READ | PAGE_WRITE;
7564         break;
7565     case 5:
7566         if (is_user) {
7567             *fsr = 1;
7568             return true;
7569         }
7570         *prot = PAGE_READ;
7571         break;
7572     case 6:
7573         *prot = PAGE_READ;
7574         break;
7575     default:
7576         /* Bad permission.  */
7577         *fsr = 1;
7578         return true;
7579     }
7580     *prot |= PAGE_EXEC;
7581     return false;
7582 }
7583
7584 /* get_phys_addr - get the physical address for this virtual address
7585  *
7586  * Find the physical address corresponding to the given virtual address,
7587  * by doing a translation table walk on MMU based systems or using the
7588  * MPU state on MPU based systems.
7589  *
7590  * Returns false if the translation was successful. Otherwise, phys_ptr, attrs,
7591  * prot and page_size may not be filled in, and the populated fsr value provides
7592  * information on why the translation aborted, in the format of a
7593  * DFSR/IFSR fault register, with the following caveats:
7594  *  * we honour the short vs long DFSR format differences.
7595  *  * the WnR bit is never set (the caller must do this).
7596  *  * for PSMAv5 based systems we don't bother to return a full FSR format
7597  *    value.
7598  *
7599  * @env: CPUARMState
7600  * @address: virtual address to get physical address for
7601  * @access_type: 0 for read, 1 for write, 2 for execute
7602  * @mmu_idx: MMU index indicating required translation regime
7603  * @phys_ptr: set to the physical address corresponding to the virtual address
7604  * @attrs: set to the memory transaction attributes to use
7605  * @prot: set to the permissions for the page containing phys_ptr
7606  * @page_size: set to the size of the page containing phys_ptr
7607  * @fsr: set to the DFSR/IFSR value on failure
7608  */
7609 static bool get_phys_addr(CPUARMState *env, target_ulong address,
7610                           int access_type, ARMMMUIdx mmu_idx,
7611                           hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
7612                           target_ulong *page_size, uint32_t *fsr,
7613                           ARMMMUFaultInfo *fi)
7614 {
7615     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
7616         /* Call ourselves recursively to do the stage 1 and then stage 2
7617          * translations.
7618          */
7619         if (arm_feature(env, ARM_FEATURE_EL2)) {
7620             hwaddr ipa;
7621             int s2_prot;
7622             int ret;
7623
7624             ret = get_phys_addr(env, address, access_type,
7625                                 mmu_idx + ARMMMUIdx_S1NSE0, &ipa, attrs,
7626                                 prot, page_size, fsr, fi);
7627
7628             /* If S1 fails or S2 is disabled, return early.  */
7629             if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
7630                 *phys_ptr = ipa;
7631                 return ret;
7632             }
7633
7634             /* S1 is done. Now do S2 translation.  */
7635             ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS,
7636                                      phys_ptr, attrs, &s2_prot,
7637                                      page_size, fsr, fi);
7638             fi->s2addr = ipa;
7639             /* Combine the S1 and S2 perms.  */
7640             *prot &= s2_prot;
7641             return ret;
7642         } else {
7643             /*
7644              * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
7645              */
7646             mmu_idx += ARMMMUIdx_S1NSE0;
7647         }
7648     }
7649
7650     /* The page table entries may downgrade secure to non-secure, but
7651      * cannot upgrade an non-secure translation regime's attributes
7652      * to secure.
7653      */
7654     attrs->secure = regime_is_secure(env, mmu_idx);
7655     attrs->user = regime_is_user(env, mmu_idx);
7656
7657     /* Fast Context Switch Extension. This doesn't exist at all in v8.
7658      * In v7 and earlier it affects all stage 1 translations.
7659      */
7660     if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS
7661         && !arm_feature(env, ARM_FEATURE_V8)) {
7662         if (regime_el(env, mmu_idx) == 3) {
7663             address += env->cp15.fcseidr_s;
7664         } else {
7665             address += env->cp15.fcseidr_ns;
7666         }
7667     }
7668
7669     /* pmsav7 has special handling for when MPU is disabled so call it before
7670      * the common MMU/MPU disabled check below.
7671      */
7672     if (arm_feature(env, ARM_FEATURE_MPU) &&
7673         arm_feature(env, ARM_FEATURE_V7)) {
7674         *page_size = TARGET_PAGE_SIZE;
7675         return get_phys_addr_pmsav7(env, address, access_type, mmu_idx,
7676                                     phys_ptr, prot, fsr);
7677     }
7678
7679     if (regime_translation_disabled(env, mmu_idx)) {
7680         /* MMU/MPU disabled.  */
7681         *phys_ptr = address;
7682         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
7683         *page_size = TARGET_PAGE_SIZE;
7684         return 0;
7685     }
7686
7687     if (arm_feature(env, ARM_FEATURE_MPU)) {
7688         /* Pre-v7 MPU */
7689         *page_size = TARGET_PAGE_SIZE;
7690         return get_phys_addr_pmsav5(env, address, access_type, mmu_idx,
7691                                     phys_ptr, prot, fsr);
7692     }
7693
7694     if (regime_using_lpae_format(env, mmu_idx)) {
7695         return get_phys_addr_lpae(env, address, access_type, mmu_idx, phys_ptr,
7696                                   attrs, prot, page_size, fsr, fi);
7697     } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
7698         return get_phys_addr_v6(env, address, access_type, mmu_idx, phys_ptr,
7699                                 attrs, prot, page_size, fsr, fi);
7700     } else {
7701         return get_phys_addr_v5(env, address, access_type, mmu_idx, phys_ptr,
7702                                 prot, page_size, fsr, fi);
7703     }
7704 }
7705
7706 /* Walk the page table and (if the mapping exists) add the page
7707  * to the TLB. Return false on success, or true on failure. Populate
7708  * fsr with ARM DFSR/IFSR fault register format value on failure.
7709  */
7710 bool arm_tlb_fill(CPUState *cs, vaddr address,
7711                   int access_type, int mmu_idx, uint32_t *fsr,
7712                   ARMMMUFaultInfo *fi)
7713 {
7714     ARMCPU *cpu = ARM_CPU(cs);
7715     CPUARMState *env = &cpu->env;
7716     hwaddr phys_addr;
7717     target_ulong page_size;
7718     int prot;
7719     int ret;
7720     MemTxAttrs attrs = {};
7721
7722     ret = get_phys_addr(env, address, access_type, mmu_idx, &phys_addr,
7723                         &attrs, &prot, &page_size, fsr, fi);
7724     if (!ret) {
7725         /* Map a single [sub]page.  */
7726         phys_addr &= TARGET_PAGE_MASK;
7727         address &= TARGET_PAGE_MASK;
7728         tlb_set_page_with_attrs(cs, address, phys_addr, attrs,
7729                                 prot, mmu_idx, page_size);
7730         return 0;
7731     }
7732
7733     return ret;
7734 }
7735
7736 hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr,
7737                                          MemTxAttrs *attrs)
7738 {
7739     ARMCPU *cpu = ARM_CPU(cs);
7740     CPUARMState *env = &cpu->env;
7741     hwaddr phys_addr;
7742     target_ulong page_size;
7743     int prot;
7744     bool ret;
7745     uint32_t fsr;
7746     ARMMMUFaultInfo fi = {};
7747
7748     *attrs = (MemTxAttrs) {};
7749
7750     ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env, false), &phys_addr,
7751                         attrs, &prot, &page_size, &fsr, &fi);
7752
7753     if (ret) {
7754         return -1;
7755     }
7756     return phys_addr;
7757 }
7758
7759 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
7760 {
7761     ARMCPU *cpu = arm_env_get_cpu(env);
7762
7763     switch (reg) {
7764     case 0: /* APSR */
7765         return xpsr_read(env) & 0xf8000000;
7766     case 1: /* IAPSR */
7767         return xpsr_read(env) & 0xf80001ff;
7768     case 2: /* EAPSR */
7769         return xpsr_read(env) & 0xff00fc00;
7770     case 3: /* xPSR */
7771         return xpsr_read(env) & 0xff00fdff;
7772     case 5: /* IPSR */
7773         return xpsr_read(env) & 0x000001ff;
7774     case 6: /* EPSR */
7775         return xpsr_read(env) & 0x0700fc00;
7776     case 7: /* IEPSR */
7777         return xpsr_read(env) & 0x0700edff;
7778     case 8: /* MSP */
7779         return env->v7m.current_sp ? env->v7m.other_sp : env->regs[13];
7780     case 9: /* PSP */
7781         return env->v7m.current_sp ? env->regs[13] : env->v7m.other_sp;
7782     case 16: /* PRIMASK */
7783         return (env->daif & PSTATE_I) != 0;
7784     case 17: /* BASEPRI */
7785     case 18: /* BASEPRI_MAX */
7786         return env->v7m.basepri;
7787     case 19: /* FAULTMASK */
7788         return (env->daif & PSTATE_F) != 0;
7789     case 20: /* CONTROL */
7790         return env->v7m.control;
7791     default:
7792         /* ??? For debugging only.  */
7793         cpu_abort(CPU(cpu), "Unimplemented system register read (%d)\n", reg);
7794         return 0;
7795     }
7796 }
7797
7798 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
7799 {
7800     ARMCPU *cpu = arm_env_get_cpu(env);
7801
7802     switch (reg) {
7803     case 0: /* APSR */
7804         xpsr_write(env, val, 0xf8000000);
7805         break;
7806     case 1: /* IAPSR */
7807         xpsr_write(env, val, 0xf8000000);
7808         break;
7809     case 2: /* EAPSR */
7810         xpsr_write(env, val, 0xfe00fc00);
7811         break;
7812     case 3: /* xPSR */
7813         xpsr_write(env, val, 0xfe00fc00);
7814         break;
7815     case 5: /* IPSR */
7816         /* IPSR bits are readonly.  */
7817         break;
7818     case 6: /* EPSR */
7819         xpsr_write(env, val, 0x0600fc00);
7820         break;
7821     case 7: /* IEPSR */
7822         xpsr_write(env, val, 0x0600fc00);
7823         break;
7824     case 8: /* MSP */
7825         if (env->v7m.current_sp)
7826             env->v7m.other_sp = val;
7827         else
7828             env->regs[13] = val;
7829         break;
7830     case 9: /* PSP */
7831         if (env->v7m.current_sp)
7832             env->regs[13] = val;
7833         else
7834             env->v7m.other_sp = val;
7835         break;
7836     case 16: /* PRIMASK */
7837         if (val & 1) {
7838             env->daif |= PSTATE_I;
7839         } else {
7840             env->daif &= ~PSTATE_I;
7841         }
7842         break;
7843     case 17: /* BASEPRI */
7844         env->v7m.basepri = val & 0xff;
7845         break;
7846     case 18: /* BASEPRI_MAX */
7847         val &= 0xff;
7848         if (val != 0 && (val < env->v7m.basepri || env->v7m.basepri == 0))
7849             env->v7m.basepri = val;
7850         break;
7851     case 19: /* FAULTMASK */
7852         if (val & 1) {
7853             env->daif |= PSTATE_F;
7854         } else {
7855             env->daif &= ~PSTATE_F;
7856         }
7857         break;
7858     case 20: /* CONTROL */
7859         env->v7m.control = val & 3;
7860         switch_v7m_sp(env, (val & 2) != 0);
7861         break;
7862     default:
7863         /* ??? For debugging only.  */
7864         cpu_abort(CPU(cpu), "Unimplemented system register write (%d)\n", reg);
7865         return;
7866     }
7867 }
7868
7869 #endif
7870
7871 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
7872 {
7873     /* Implement DC ZVA, which zeroes a fixed-length block of memory.
7874      * Note that we do not implement the (architecturally mandated)
7875      * alignment fault for attempts to use this on Device memory
7876      * (which matches the usual QEMU behaviour of not implementing either
7877      * alignment faults or any memory attribute handling).
7878      */
7879
7880     ARMCPU *cpu = arm_env_get_cpu(env);
7881     uint64_t blocklen = 4 << cpu->dcz_blocksize;
7882     uint64_t vaddr = vaddr_in & ~(blocklen - 1);
7883
7884 #ifndef CONFIG_USER_ONLY
7885     {
7886         /* Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than
7887          * the block size so we might have to do more than one TLB lookup.
7888          * We know that in fact for any v8 CPU the page size is at least 4K
7889          * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only
7890          * 1K as an artefact of legacy v5 subpage support being present in the
7891          * same QEMU executable.
7892          */
7893         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
7894         void *hostaddr[maxidx];
7895         int try, i;
7896         unsigned mmu_idx = cpu_mmu_index(env, false);
7897         TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
7898
7899         for (try = 0; try < 2; try++) {
7900
7901             for (i = 0; i < maxidx; i++) {
7902                 hostaddr[i] = tlb_vaddr_to_host(env,
7903                                                 vaddr + TARGET_PAGE_SIZE * i,
7904                                                 1, mmu_idx);
7905                 if (!hostaddr[i]) {
7906                     break;
7907                 }
7908             }
7909             if (i == maxidx) {
7910                 /* If it's all in the TLB it's fair game for just writing to;
7911                  * we know we don't need to update dirty status, etc.
7912                  */
7913                 for (i = 0; i < maxidx - 1; i++) {
7914                     memset(hostaddr[i], 0, TARGET_PAGE_SIZE);
7915                 }
7916                 memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE));
7917                 return;
7918             }
7919             /* OK, try a store and see if we can populate the tlb. This
7920              * might cause an exception if the memory isn't writable,
7921              * in which case we will longjmp out of here. We must for
7922              * this purpose use the actual register value passed to us
7923              * so that we get the fault address right.
7924              */
7925             helper_ret_stb_mmu(env, vaddr_in, 0, oi, GETRA());
7926             /* Now we can populate the other TLB entries, if any */
7927             for (i = 0; i < maxidx; i++) {
7928                 uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
7929                 if (va != (vaddr_in & TARGET_PAGE_MASK)) {
7930                     helper_ret_stb_mmu(env, va, 0, oi, GETRA());
7931                 }
7932             }
7933         }
7934
7935         /* Slow path (probably attempt to do this to an I/O device or
7936          * similar, or clearing of a block of code we have translations
7937          * cached for). Just do a series of byte writes as the architecture
7938          * demands. It's not worth trying to use a cpu_physical_memory_map(),
7939          * memset(), unmap() sequence here because:
7940          *  + we'd need to account for the blocksize being larger than a page
7941          *  + the direct-RAM access case is almost always going to be dealt
7942          *    with in the fastpath code above, so there's no speed benefit
7943          *  + we would have to deal with the map returning NULL because the
7944          *    bounce buffer was in use
7945          */
7946         for (i = 0; i < blocklen; i++) {
7947             helper_ret_stb_mmu(env, vaddr + i, 0, oi, GETRA());
7948         }
7949     }
7950 #else
7951     memset(g2h(vaddr), 0, blocklen);
7952 #endif
7953 }
7954
7955 /* Note that signed overflow is undefined in C.  The following routines are
7956    careful to use unsigned types where modulo arithmetic is required.
7957    Failure to do so _will_ break on newer gcc.  */
7958
7959 /* Signed saturating arithmetic.  */
7960
7961 /* Perform 16-bit signed saturating addition.  */
7962 static inline uint16_t add16_sat(uint16_t a, uint16_t b)
7963 {
7964     uint16_t res;
7965
7966     res = a + b;
7967     if (((res ^ a) & 0x8000) && !((a ^ b) & 0x8000)) {
7968         if (a & 0x8000)
7969             res = 0x8000;
7970         else
7971             res = 0x7fff;
7972     }
7973     return res;
7974 }
7975
7976 /* Perform 8-bit signed saturating addition.  */
7977 static inline uint8_t add8_sat(uint8_t a, uint8_t b)
7978 {
7979     uint8_t res;
7980
7981     res = a + b;
7982     if (((res ^ a) & 0x80) && !((a ^ b) & 0x80)) {
7983         if (a & 0x80)
7984             res = 0x80;
7985         else
7986             res = 0x7f;
7987     }
7988     return res;
7989 }
7990
7991 /* Perform 16-bit signed saturating subtraction.  */
7992 static inline uint16_t sub16_sat(uint16_t a, uint16_t b)
7993 {
7994     uint16_t res;
7995
7996     res = a - b;
7997     if (((res ^ a) & 0x8000) && ((a ^ b) & 0x8000)) {
7998         if (a & 0x8000)
7999             res = 0x8000;
8000         else
8001             res = 0x7fff;
8002     }
8003     return res;
8004 }
8005
8006 /* Perform 8-bit signed saturating subtraction.  */
8007 static inline uint8_t sub8_sat(uint8_t a, uint8_t b)
8008 {
8009     uint8_t res;
8010
8011     res = a - b;
8012     if (((res ^ a) & 0x80) && ((a ^ b) & 0x80)) {
8013         if (a & 0x80)
8014             res = 0x80;
8015         else
8016             res = 0x7f;
8017     }
8018     return res;
8019 }
8020
8021 #define ADD16(a, b, n) RESULT(add16_sat(a, b), n, 16);
8022 #define SUB16(a, b, n) RESULT(sub16_sat(a, b), n, 16);
8023 #define ADD8(a, b, n)  RESULT(add8_sat(a, b), n, 8);
8024 #define SUB8(a, b, n)  RESULT(sub8_sat(a, b), n, 8);
8025 #define PFX q
8026
8027 #include "op_addsub.h"
8028
8029 /* Unsigned saturating arithmetic.  */
8030 static inline uint16_t add16_usat(uint16_t a, uint16_t b)
8031 {
8032     uint16_t res;
8033     res = a + b;
8034     if (res < a)
8035         res = 0xffff;
8036     return res;
8037 }
8038
8039 static inline uint16_t sub16_usat(uint16_t a, uint16_t b)
8040 {
8041     if (a > b)
8042         return a - b;
8043     else
8044         return 0;
8045 }
8046
8047 static inline uint8_t add8_usat(uint8_t a, uint8_t b)
8048 {
8049     uint8_t res;
8050     res = a + b;
8051     if (res < a)
8052         res = 0xff;
8053     return res;
8054 }
8055
8056 static inline uint8_t sub8_usat(uint8_t a, uint8_t b)
8057 {
8058     if (a > b)
8059         return a - b;
8060     else
8061         return 0;
8062 }
8063
8064 #define ADD16(a, b, n) RESULT(add16_usat(a, b), n, 16);
8065 #define SUB16(a, b, n) RESULT(sub16_usat(a, b), n, 16);
8066 #define ADD8(a, b, n)  RESULT(add8_usat(a, b), n, 8);
8067 #define SUB8(a, b, n)  RESULT(sub8_usat(a, b), n, 8);
8068 #define PFX uq
8069
8070 #include "op_addsub.h"
8071
8072 /* Signed modulo arithmetic.  */
8073 #define SARITH16(a, b, n, op) do { \
8074     int32_t sum; \
8075     sum = (int32_t)(int16_t)(a) op (int32_t)(int16_t)(b); \
8076     RESULT(sum, n, 16); \
8077     if (sum >= 0) \
8078         ge |= 3 << (n * 2); \
8079     } while(0)
8080
8081 #define SARITH8(a, b, n, op) do { \
8082     int32_t sum; \
8083     sum = (int32_t)(int8_t)(a) op (int32_t)(int8_t)(b); \
8084     RESULT(sum, n, 8); \
8085     if (sum >= 0) \
8086         ge |= 1 << n; \
8087     } while(0)
8088
8089
8090 #define ADD16(a, b, n) SARITH16(a, b, n, +)
8091 #define SUB16(a, b, n) SARITH16(a, b, n, -)
8092 #define ADD8(a, b, n)  SARITH8(a, b, n, +)
8093 #define SUB8(a, b, n)  SARITH8(a, b, n, -)
8094 #define PFX s
8095 #define ARITH_GE
8096
8097 #include "op_addsub.h"
8098
8099 /* Unsigned modulo arithmetic.  */
8100 #define ADD16(a, b, n) do { \
8101     uint32_t sum; \
8102     sum = (uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b); \
8103     RESULT(sum, n, 16); \
8104     if ((sum >> 16) == 1) \
8105         ge |= 3 << (n * 2); \
8106     } while(0)
8107
8108 #define ADD8(a, b, n) do { \
8109     uint32_t sum; \
8110     sum = (uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b); \
8111     RESULT(sum, n, 8); \
8112     if ((sum >> 8) == 1) \
8113         ge |= 1 << n; \
8114     } while(0)
8115
8116 #define SUB16(a, b, n) do { \
8117     uint32_t sum; \
8118     sum = (uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b); \
8119     RESULT(sum, n, 16); \
8120     if ((sum >> 16) == 0) \
8121         ge |= 3 << (n * 2); \
8122     } while(0)
8123
8124 #define SUB8(a, b, n) do { \
8125     uint32_t sum; \
8126     sum = (uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b); \
8127     RESULT(sum, n, 8); \
8128     if ((sum >> 8) == 0) \
8129         ge |= 1 << n; \
8130     } while(0)
8131
8132 #define PFX u
8133 #define ARITH_GE
8134
8135 #include "op_addsub.h"
8136
8137 /* Halved signed arithmetic.  */
8138 #define ADD16(a, b, n) \
8139   RESULT(((int32_t)(int16_t)(a) + (int32_t)(int16_t)(b)) >> 1, n, 16)
8140 #define SUB16(a, b, n) \
8141   RESULT(((int32_t)(int16_t)(a) - (int32_t)(int16_t)(b)) >> 1, n, 16)
8142 #define ADD8(a, b, n) \
8143   RESULT(((int32_t)(int8_t)(a) + (int32_t)(int8_t)(b)) >> 1, n, 8)
8144 #define SUB8(a, b, n) \
8145   RESULT(((int32_t)(int8_t)(a) - (int32_t)(int8_t)(b)) >> 1, n, 8)
8146 #define PFX sh
8147
8148 #include "op_addsub.h"
8149
8150 /* Halved unsigned arithmetic.  */
8151 #define ADD16(a, b, n) \
8152   RESULT(((uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b)) >> 1, n, 16)
8153 #define SUB16(a, b, n) \
8154   RESULT(((uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b)) >> 1, n, 16)
8155 #define ADD8(a, b, n) \
8156   RESULT(((uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b)) >> 1, n, 8)
8157 #define SUB8(a, b, n) \
8158   RESULT(((uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b)) >> 1, n, 8)
8159 #define PFX uh
8160
8161 #include "op_addsub.h"
8162
8163 static inline uint8_t do_usad(uint8_t a, uint8_t b)
8164 {
8165     if (a > b)
8166         return a - b;
8167     else
8168         return b - a;
8169 }
8170
8171 /* Unsigned sum of absolute byte differences.  */
8172 uint32_t HELPER(usad8)(uint32_t a, uint32_t b)
8173 {
8174     uint32_t sum;
8175     sum = do_usad(a, b);
8176     sum += do_usad(a >> 8, b >> 8);
8177     sum += do_usad(a >> 16, b >>16);
8178     sum += do_usad(a >> 24, b >> 24);
8179     return sum;
8180 }
8181
8182 /* For ARMv6 SEL instruction.  */
8183 uint32_t HELPER(sel_flags)(uint32_t flags, uint32_t a, uint32_t b)
8184 {
8185     uint32_t mask;
8186
8187     mask = 0;
8188     if (flags & 1)
8189         mask |= 0xff;
8190     if (flags & 2)
8191         mask |= 0xff00;
8192     if (flags & 4)
8193         mask |= 0xff0000;
8194     if (flags & 8)
8195         mask |= 0xff000000;
8196     return (a & mask) | (b & ~mask);
8197 }
8198
8199 /* VFP support.  We follow the convention used for VFP instructions:
8200    Single precision routines have a "s" suffix, double precision a
8201    "d" suffix.  */
8202
8203 /* Convert host exception flags to vfp form.  */
8204 static inline int vfp_exceptbits_from_host(int host_bits)
8205 {
8206     int target_bits = 0;
8207
8208     if (host_bits & float_flag_invalid)
8209         target_bits |= 1;
8210     if (host_bits & float_flag_divbyzero)
8211         target_bits |= 2;
8212     if (host_bits & float_flag_overflow)
8213         target_bits |= 4;
8214     if (host_bits & (float_flag_underflow | float_flag_output_denormal))
8215         target_bits |= 8;
8216     if (host_bits & float_flag_inexact)
8217         target_bits |= 0x10;
8218     if (host_bits & float_flag_input_denormal)
8219         target_bits |= 0x80;
8220     return target_bits;
8221 }
8222
8223 uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env)
8224 {
8225     int i;
8226     uint32_t fpscr;
8227
8228     fpscr = (env->vfp.xregs[ARM_VFP_FPSCR] & 0xffc8ffff)
8229             | (env->vfp.vec_len << 16)
8230             | (env->vfp.vec_stride << 20);
8231     i = get_float_exception_flags(&env->vfp.fp_status);
8232     i |= get_float_exception_flags(&env->vfp.standard_fp_status);
8233     fpscr |= vfp_exceptbits_from_host(i);
8234     return fpscr;
8235 }
8236
8237 uint32_t vfp_get_fpscr(CPUARMState *env)
8238 {
8239     return HELPER(vfp_get_fpscr)(env);
8240 }
8241
8242 /* Convert vfp exception flags to target form.  */
8243 static inline int vfp_exceptbits_to_host(int target_bits)
8244 {
8245     int host_bits = 0;
8246
8247     if (target_bits & 1)
8248         host_bits |= float_flag_invalid;
8249     if (target_bits & 2)
8250         host_bits |= float_flag_divbyzero;
8251     if (target_bits & 4)
8252         host_bits |= float_flag_overflow;
8253     if (target_bits & 8)
8254         host_bits |= float_flag_underflow;
8255     if (target_bits & 0x10)
8256         host_bits |= float_flag_inexact;
8257     if (target_bits & 0x80)
8258         host_bits |= float_flag_input_denormal;
8259     return host_bits;
8260 }
8261
8262 void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val)
8263 {
8264     int i;
8265     uint32_t changed;
8266
8267     changed = env->vfp.xregs[ARM_VFP_FPSCR];
8268     env->vfp.xregs[ARM_VFP_FPSCR] = (val & 0xffc8ffff);
8269     env->vfp.vec_len = (val >> 16) & 7;
8270     env->vfp.vec_stride = (val >> 20) & 3;
8271
8272     changed ^= val;
8273     if (changed & (3 << 22)) {
8274         i = (val >> 22) & 3;
8275         switch (i) {
8276         case FPROUNDING_TIEEVEN:
8277             i = float_round_nearest_even;
8278             break;
8279         case FPROUNDING_POSINF:
8280             i = float_round_up;
8281             break;
8282         case FPROUNDING_NEGINF:
8283             i = float_round_down;
8284             break;
8285         case FPROUNDING_ZERO:
8286             i = float_round_to_zero;
8287             break;
8288         }
8289         set_float_rounding_mode(i, &env->vfp.fp_status);
8290     }
8291     if (changed & (1 << 24)) {
8292         set_flush_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
8293         set_flush_inputs_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
8294     }
8295     if (changed & (1 << 25))
8296         set_default_nan_mode((val & (1 << 25)) != 0, &env->vfp.fp_status);
8297
8298     i = vfp_exceptbits_to_host(val);
8299     set_float_exception_flags(i, &env->vfp.fp_status);
8300     set_float_exception_flags(0, &env->vfp.standard_fp_status);
8301 }
8302
8303 void vfp_set_fpscr(CPUARMState *env, uint32_t val)
8304 {
8305     HELPER(vfp_set_fpscr)(env, val);
8306 }
8307
8308 #define VFP_HELPER(name, p) HELPER(glue(glue(vfp_,name),p))
8309
8310 #define VFP_BINOP(name) \
8311 float32 VFP_HELPER(name, s)(float32 a, float32 b, void *fpstp) \
8312 { \
8313     float_status *fpst = fpstp; \
8314     return float32_ ## name(a, b, fpst); \
8315 } \
8316 float64 VFP_HELPER(name, d)(float64 a, float64 b, void *fpstp) \
8317 { \
8318     float_status *fpst = fpstp; \
8319     return float64_ ## name(a, b, fpst); \
8320 }
8321 VFP_BINOP(add)
8322 VFP_BINOP(sub)
8323 VFP_BINOP(mul)
8324 VFP_BINOP(div)
8325 VFP_BINOP(min)
8326 VFP_BINOP(max)
8327 VFP_BINOP(minnum)
8328 VFP_BINOP(maxnum)
8329 #undef VFP_BINOP
8330
8331 float32 VFP_HELPER(neg, s)(float32 a)
8332 {
8333     return float32_chs(a);
8334 }
8335
8336 float64 VFP_HELPER(neg, d)(float64 a)
8337 {
8338     return float64_chs(a);
8339 }
8340
8341 float32 VFP_HELPER(abs, s)(float32 a)
8342 {
8343     return float32_abs(a);
8344 }
8345
8346 float64 VFP_HELPER(abs, d)(float64 a)
8347 {
8348     return float64_abs(a);
8349 }
8350
8351 float32 VFP_HELPER(sqrt, s)(float32 a, CPUARMState *env)
8352 {
8353     return float32_sqrt(a, &env->vfp.fp_status);
8354 }
8355
8356 float64 VFP_HELPER(sqrt, d)(float64 a, CPUARMState *env)
8357 {
8358     return float64_sqrt(a, &env->vfp.fp_status);
8359 }
8360
8361 /* XXX: check quiet/signaling case */
8362 #define DO_VFP_cmp(p, type) \
8363 void VFP_HELPER(cmp, p)(type a, type b, CPUARMState *env)  \
8364 { \
8365     uint32_t flags; \
8366     switch(type ## _compare_quiet(a, b, &env->vfp.fp_status)) { \
8367     case 0: flags = 0x6; break; \
8368     case -1: flags = 0x8; break; \
8369     case 1: flags = 0x2; break; \
8370     default: case 2: flags = 0x3; break; \
8371     } \
8372     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
8373         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
8374 } \
8375 void VFP_HELPER(cmpe, p)(type a, type b, CPUARMState *env) \
8376 { \
8377     uint32_t flags; \
8378     switch(type ## _compare(a, b, &env->vfp.fp_status)) { \
8379     case 0: flags = 0x6; break; \
8380     case -1: flags = 0x8; break; \
8381     case 1: flags = 0x2; break; \
8382     default: case 2: flags = 0x3; break; \
8383     } \
8384     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
8385         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
8386 }
8387 DO_VFP_cmp(s, float32)
8388 DO_VFP_cmp(d, float64)
8389 #undef DO_VFP_cmp
8390
8391 /* Integer to float and float to integer conversions */
8392
8393 #define CONV_ITOF(name, fsz, sign) \
8394     float##fsz HELPER(name)(uint32_t x, void *fpstp) \
8395 { \
8396     float_status *fpst = fpstp; \
8397     return sign##int32_to_##float##fsz((sign##int32_t)x, fpst); \
8398 }
8399
8400 #define CONV_FTOI(name, fsz, sign, round) \
8401 uint32_t HELPER(name)(float##fsz x, void *fpstp) \
8402 { \
8403     float_status *fpst = fpstp; \
8404     if (float##fsz##_is_any_nan(x)) { \
8405         float_raise(float_flag_invalid, fpst); \
8406         return 0; \
8407     } \
8408     return float##fsz##_to_##sign##int32##round(x, fpst); \
8409 }
8410
8411 #define FLOAT_CONVS(name, p, fsz, sign) \
8412 CONV_ITOF(vfp_##name##to##p, fsz, sign) \
8413 CONV_FTOI(vfp_to##name##p, fsz, sign, ) \
8414 CONV_FTOI(vfp_to##name##z##p, fsz, sign, _round_to_zero)
8415
8416 FLOAT_CONVS(si, s, 32, )
8417 FLOAT_CONVS(si, d, 64, )
8418 FLOAT_CONVS(ui, s, 32, u)
8419 FLOAT_CONVS(ui, d, 64, u)
8420
8421 #undef CONV_ITOF
8422 #undef CONV_FTOI
8423 #undef FLOAT_CONVS
8424
8425 /* floating point conversion */
8426 float64 VFP_HELPER(fcvtd, s)(float32 x, CPUARMState *env)
8427 {
8428     float64 r = float32_to_float64(x, &env->vfp.fp_status);
8429     /* ARM requires that S<->D conversion of any kind of NaN generates
8430      * a quiet NaN by forcing the most significant frac bit to 1.
8431      */
8432     return float64_maybe_silence_nan(r);
8433 }
8434
8435 float32 VFP_HELPER(fcvts, d)(float64 x, CPUARMState *env)
8436 {
8437     float32 r =  float64_to_float32(x, &env->vfp.fp_status);
8438     /* ARM requires that S<->D conversion of any kind of NaN generates
8439      * a quiet NaN by forcing the most significant frac bit to 1.
8440      */
8441     return float32_maybe_silence_nan(r);
8442 }
8443
8444 /* VFP3 fixed point conversion.  */
8445 #define VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype) \
8446 float##fsz HELPER(vfp_##name##to##p)(uint##isz##_t  x, uint32_t shift, \
8447                                      void *fpstp) \
8448 { \
8449     float_status *fpst = fpstp; \
8450     float##fsz tmp; \
8451     tmp = itype##_to_##float##fsz(x, fpst); \
8452     return float##fsz##_scalbn(tmp, -(int)shift, fpst); \
8453 }
8454
8455 /* Notice that we want only input-denormal exception flags from the
8456  * scalbn operation: the other possible flags (overflow+inexact if
8457  * we overflow to infinity, output-denormal) aren't correct for the
8458  * complete scale-and-convert operation.
8459  */
8460 #define VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, round) \
8461 uint##isz##_t HELPER(vfp_to##name##p##round)(float##fsz x, \
8462                                              uint32_t shift, \
8463                                              void *fpstp) \
8464 { \
8465     float_status *fpst = fpstp; \
8466     int old_exc_flags = get_float_exception_flags(fpst); \
8467     float##fsz tmp; \
8468     if (float##fsz##_is_any_nan(x)) { \
8469         float_raise(float_flag_invalid, fpst); \
8470         return 0; \
8471     } \
8472     tmp = float##fsz##_scalbn(x, shift, fpst); \
8473     old_exc_flags |= get_float_exception_flags(fpst) \
8474         & float_flag_input_denormal; \
8475     set_float_exception_flags(old_exc_flags, fpst); \
8476     return float##fsz##_to_##itype##round(tmp, fpst); \
8477 }
8478
8479 #define VFP_CONV_FIX(name, p, fsz, isz, itype)                   \
8480 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
8481 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, _round_to_zero) \
8482 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
8483
8484 #define VFP_CONV_FIX_A64(name, p, fsz, isz, itype)               \
8485 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
8486 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
8487
8488 VFP_CONV_FIX(sh, d, 64, 64, int16)
8489 VFP_CONV_FIX(sl, d, 64, 64, int32)
8490 VFP_CONV_FIX_A64(sq, d, 64, 64, int64)
8491 VFP_CONV_FIX(uh, d, 64, 64, uint16)
8492 VFP_CONV_FIX(ul, d, 64, 64, uint32)
8493 VFP_CONV_FIX_A64(uq, d, 64, 64, uint64)
8494 VFP_CONV_FIX(sh, s, 32, 32, int16)
8495 VFP_CONV_FIX(sl, s, 32, 32, int32)
8496 VFP_CONV_FIX_A64(sq, s, 32, 64, int64)
8497 VFP_CONV_FIX(uh, s, 32, 32, uint16)
8498 VFP_CONV_FIX(ul, s, 32, 32, uint32)
8499 VFP_CONV_FIX_A64(uq, s, 32, 64, uint64)
8500 #undef VFP_CONV_FIX
8501 #undef VFP_CONV_FIX_FLOAT
8502 #undef VFP_CONV_FLOAT_FIX_ROUND
8503
8504 /* Set the current fp rounding mode and return the old one.
8505  * The argument is a softfloat float_round_ value.
8506  */
8507 uint32_t HELPER(set_rmode)(uint32_t rmode, CPUARMState *env)
8508 {
8509     float_status *fp_status = &env->vfp.fp_status;
8510
8511     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
8512     set_float_rounding_mode(rmode, fp_status);
8513
8514     return prev_rmode;
8515 }
8516
8517 /* Set the current fp rounding mode in the standard fp status and return
8518  * the old one. This is for NEON instructions that need to change the
8519  * rounding mode but wish to use the standard FPSCR values for everything
8520  * else. Always set the rounding mode back to the correct value after
8521  * modifying it.
8522  * The argument is a softfloat float_round_ value.
8523  */
8524 uint32_t HELPER(set_neon_rmode)(uint32_t rmode, CPUARMState *env)
8525 {
8526     float_status *fp_status = &env->vfp.standard_fp_status;
8527
8528     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
8529     set_float_rounding_mode(rmode, fp_status);
8530
8531     return prev_rmode;
8532 }
8533
8534 /* Half precision conversions.  */
8535 static float32 do_fcvt_f16_to_f32(uint32_t a, CPUARMState *env, float_status *s)
8536 {
8537     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8538     float32 r = float16_to_float32(make_float16(a), ieee, s);
8539     if (ieee) {
8540         return float32_maybe_silence_nan(r);
8541     }
8542     return r;
8543 }
8544
8545 static uint32_t do_fcvt_f32_to_f16(float32 a, CPUARMState *env, float_status *s)
8546 {
8547     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8548     float16 r = float32_to_float16(a, ieee, s);
8549     if (ieee) {
8550         r = float16_maybe_silence_nan(r);
8551     }
8552     return float16_val(r);
8553 }
8554
8555 float32 HELPER(neon_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
8556 {
8557     return do_fcvt_f16_to_f32(a, env, &env->vfp.standard_fp_status);
8558 }
8559
8560 uint32_t HELPER(neon_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
8561 {
8562     return do_fcvt_f32_to_f16(a, env, &env->vfp.standard_fp_status);
8563 }
8564
8565 float32 HELPER(vfp_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
8566 {
8567     return do_fcvt_f16_to_f32(a, env, &env->vfp.fp_status);
8568 }
8569
8570 uint32_t HELPER(vfp_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
8571 {
8572     return do_fcvt_f32_to_f16(a, env, &env->vfp.fp_status);
8573 }
8574
8575 float64 HELPER(vfp_fcvt_f16_to_f64)(uint32_t a, CPUARMState *env)
8576 {
8577     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8578     float64 r = float16_to_float64(make_float16(a), ieee, &env->vfp.fp_status);
8579     if (ieee) {
8580         return float64_maybe_silence_nan(r);
8581     }
8582     return r;
8583 }
8584
8585 uint32_t HELPER(vfp_fcvt_f64_to_f16)(float64 a, CPUARMState *env)
8586 {
8587     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8588     float16 r = float64_to_float16(a, ieee, &env->vfp.fp_status);
8589     if (ieee) {
8590         r = float16_maybe_silence_nan(r);
8591     }
8592     return float16_val(r);
8593 }
8594
8595 #define float32_two make_float32(0x40000000)
8596 #define float32_three make_float32(0x40400000)
8597 #define float32_one_point_five make_float32(0x3fc00000)
8598
8599 float32 HELPER(recps_f32)(float32 a, float32 b, CPUARMState *env)
8600 {
8601     float_status *s = &env->vfp.standard_fp_status;
8602     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
8603         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
8604         if (!(float32_is_zero(a) || float32_is_zero(b))) {
8605             float_raise(float_flag_input_denormal, s);
8606         }
8607         return float32_two;
8608     }
8609     return float32_sub(float32_two, float32_mul(a, b, s), s);
8610 }
8611
8612 float32 HELPER(rsqrts_f32)(float32 a, float32 b, CPUARMState *env)
8613 {
8614     float_status *s = &env->vfp.standard_fp_status;
8615     float32 product;
8616     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
8617         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
8618         if (!(float32_is_zero(a) || float32_is_zero(b))) {
8619             float_raise(float_flag_input_denormal, s);
8620         }
8621         return float32_one_point_five;
8622     }
8623     product = float32_mul(a, b, s);
8624     return float32_div(float32_sub(float32_three, product, s), float32_two, s);
8625 }
8626
8627 /* NEON helpers.  */
8628
8629 /* Constants 256 and 512 are used in some helpers; we avoid relying on
8630  * int->float conversions at run-time.  */
8631 #define float64_256 make_float64(0x4070000000000000LL)
8632 #define float64_512 make_float64(0x4080000000000000LL)
8633 #define float32_maxnorm make_float32(0x7f7fffff)
8634 #define float64_maxnorm make_float64(0x7fefffffffffffffLL)
8635
8636 /* Reciprocal functions
8637  *
8638  * The algorithm that must be used to calculate the estimate
8639  * is specified by the ARM ARM, see FPRecipEstimate()
8640  */
8641
8642 static float64 recip_estimate(float64 a, float_status *real_fp_status)
8643 {
8644     /* These calculations mustn't set any fp exception flags,
8645      * so we use a local copy of the fp_status.
8646      */
8647     float_status dummy_status = *real_fp_status;
8648     float_status *s = &dummy_status;
8649     /* q = (int)(a * 512.0) */
8650     float64 q = float64_mul(float64_512, a, s);
8651     int64_t q_int = float64_to_int64_round_to_zero(q, s);
8652
8653     /* r = 1.0 / (((double)q + 0.5) / 512.0) */
8654     q = int64_to_float64(q_int, s);
8655     q = float64_add(q, float64_half, s);
8656     q = float64_div(q, float64_512, s);
8657     q = float64_div(float64_one, q, s);
8658
8659     /* s = (int)(256.0 * r + 0.5) */
8660     q = float64_mul(q, float64_256, s);
8661     q = float64_add(q, float64_half, s);
8662     q_int = float64_to_int64_round_to_zero(q, s);
8663
8664     /* return (double)s / 256.0 */
8665     return float64_div(int64_to_float64(q_int, s), float64_256, s);
8666 }
8667
8668 /* Common wrapper to call recip_estimate */
8669 static float64 call_recip_estimate(float64 num, int off, float_status *fpst)
8670 {
8671     uint64_t val64 = float64_val(num);
8672     uint64_t frac = extract64(val64, 0, 52);
8673     int64_t exp = extract64(val64, 52, 11);
8674     uint64_t sbit;
8675     float64 scaled, estimate;
8676
8677     /* Generate the scaled number for the estimate function */
8678     if (exp == 0) {
8679         if (extract64(frac, 51, 1) == 0) {
8680             exp = -1;
8681             frac = extract64(frac, 0, 50) << 2;
8682         } else {
8683             frac = extract64(frac, 0, 51) << 1;
8684         }
8685     }
8686
8687     /* scaled = '0' : '01111111110' : fraction<51:44> : Zeros(44); */
8688     scaled = make_float64((0x3feULL << 52)
8689                           | extract64(frac, 44, 8) << 44);
8690
8691     estimate = recip_estimate(scaled, fpst);
8692
8693     /* Build new result */
8694     val64 = float64_val(estimate);
8695     sbit = 0x8000000000000000ULL & val64;
8696     exp = off - exp;
8697     frac = extract64(val64, 0, 52);
8698
8699     if (exp == 0) {
8700         frac = 1ULL << 51 | extract64(frac, 1, 51);
8701     } else if (exp == -1) {
8702         frac = 1ULL << 50 | extract64(frac, 2, 50);
8703         exp = 0;
8704     }
8705
8706     return make_float64(sbit | (exp << 52) | frac);
8707 }
8708
8709 static bool round_to_inf(float_status *fpst, bool sign_bit)
8710 {
8711     switch (fpst->float_rounding_mode) {
8712     case float_round_nearest_even: /* Round to Nearest */
8713         return true;
8714     case float_round_up: /* Round to +Inf */
8715         return !sign_bit;
8716     case float_round_down: /* Round to -Inf */
8717         return sign_bit;
8718     case float_round_to_zero: /* Round to Zero */
8719         return false;
8720     }
8721
8722     g_assert_not_reached();
8723 }
8724
8725 float32 HELPER(recpe_f32)(float32 input, void *fpstp)
8726 {
8727     float_status *fpst = fpstp;
8728     float32 f32 = float32_squash_input_denormal(input, fpst);
8729     uint32_t f32_val = float32_val(f32);
8730     uint32_t f32_sbit = 0x80000000ULL & f32_val;
8731     int32_t f32_exp = extract32(f32_val, 23, 8);
8732     uint32_t f32_frac = extract32(f32_val, 0, 23);
8733     float64 f64, r64;
8734     uint64_t r64_val;
8735     int64_t r64_exp;
8736     uint64_t r64_frac;
8737
8738     if (float32_is_any_nan(f32)) {
8739         float32 nan = f32;
8740         if (float32_is_signaling_nan(f32)) {
8741             float_raise(float_flag_invalid, fpst);
8742             nan = float32_maybe_silence_nan(f32);
8743         }
8744         if (fpst->default_nan_mode) {
8745             nan =  float32_default_nan;
8746         }
8747         return nan;
8748     } else if (float32_is_infinity(f32)) {
8749         return float32_set_sign(float32_zero, float32_is_neg(f32));
8750     } else if (float32_is_zero(f32)) {
8751         float_raise(float_flag_divbyzero, fpst);
8752         return float32_set_sign(float32_infinity, float32_is_neg(f32));
8753     } else if ((f32_val & ~(1ULL << 31)) < (1ULL << 21)) {
8754         /* Abs(value) < 2.0^-128 */
8755         float_raise(float_flag_overflow | float_flag_inexact, fpst);
8756         if (round_to_inf(fpst, f32_sbit)) {
8757             return float32_set_sign(float32_infinity, float32_is_neg(f32));
8758         } else {
8759             return float32_set_sign(float32_maxnorm, float32_is_neg(f32));
8760         }
8761     } else if (f32_exp >= 253 && fpst->flush_to_zero) {
8762         float_raise(float_flag_underflow, fpst);
8763         return float32_set_sign(float32_zero, float32_is_neg(f32));
8764     }
8765
8766
8767     f64 = make_float64(((int64_t)(f32_exp) << 52) | (int64_t)(f32_frac) << 29);
8768     r64 = call_recip_estimate(f64, 253, fpst);
8769     r64_val = float64_val(r64);
8770     r64_exp = extract64(r64_val, 52, 11);
8771     r64_frac = extract64(r64_val, 0, 52);
8772
8773     /* result = sign : result_exp<7:0> : fraction<51:29>; */
8774     return make_float32(f32_sbit |
8775                         (r64_exp & 0xff) << 23 |
8776                         extract64(r64_frac, 29, 24));
8777 }
8778
8779 float64 HELPER(recpe_f64)(float64 input, void *fpstp)
8780 {
8781     float_status *fpst = fpstp;
8782     float64 f64 = float64_squash_input_denormal(input, fpst);
8783     uint64_t f64_val = float64_val(f64);
8784     uint64_t f64_sbit = 0x8000000000000000ULL & f64_val;
8785     int64_t f64_exp = extract64(f64_val, 52, 11);
8786     float64 r64;
8787     uint64_t r64_val;
8788     int64_t r64_exp;
8789     uint64_t r64_frac;
8790
8791     /* Deal with any special cases */
8792     if (float64_is_any_nan(f64)) {
8793         float64 nan = f64;
8794         if (float64_is_signaling_nan(f64)) {
8795             float_raise(float_flag_invalid, fpst);
8796             nan = float64_maybe_silence_nan(f64);
8797         }
8798         if (fpst->default_nan_mode) {
8799             nan =  float64_default_nan;
8800         }
8801         return nan;
8802     } else if (float64_is_infinity(f64)) {
8803         return float64_set_sign(float64_zero, float64_is_neg(f64));
8804     } else if (float64_is_zero(f64)) {
8805         float_raise(float_flag_divbyzero, fpst);
8806         return float64_set_sign(float64_infinity, float64_is_neg(f64));
8807     } else if ((f64_val & ~(1ULL << 63)) < (1ULL << 50)) {
8808         /* Abs(value) < 2.0^-1024 */
8809         float_raise(float_flag_overflow | float_flag_inexact, fpst);
8810         if (round_to_inf(fpst, f64_sbit)) {
8811             return float64_set_sign(float64_infinity, float64_is_neg(f64));
8812         } else {
8813             return float64_set_sign(float64_maxnorm, float64_is_neg(f64));
8814         }
8815     } else if (f64_exp >= 2045 && fpst->flush_to_zero) {
8816         float_raise(float_flag_underflow, fpst);
8817         return float64_set_sign(float64_zero, float64_is_neg(f64));
8818     }
8819
8820     r64 = call_recip_estimate(f64, 2045, fpst);
8821     r64_val = float64_val(r64);
8822     r64_exp = extract64(r64_val, 52, 11);
8823     r64_frac = extract64(r64_val, 0, 52);
8824
8825     /* result = sign : result_exp<10:0> : fraction<51:0> */
8826     return make_float64(f64_sbit |
8827                         ((r64_exp & 0x7ff) << 52) |
8828                         r64_frac);
8829 }
8830
8831 /* The algorithm that must be used to calculate the estimate
8832  * is specified by the ARM ARM.
8833  */
8834 static float64 recip_sqrt_estimate(float64 a, float_status *real_fp_status)
8835 {
8836     /* These calculations mustn't set any fp exception flags,
8837      * so we use a local copy of the fp_status.
8838      */
8839     float_status dummy_status = *real_fp_status;
8840     float_status *s = &dummy_status;
8841     float64 q;
8842     int64_t q_int;
8843
8844     if (float64_lt(a, float64_half, s)) {
8845         /* range 0.25 <= a < 0.5 */
8846
8847         /* a in units of 1/512 rounded down */
8848         /* q0 = (int)(a * 512.0);  */
8849         q = float64_mul(float64_512, a, s);
8850         q_int = float64_to_int64_round_to_zero(q, s);
8851
8852         /* reciprocal root r */
8853         /* r = 1.0 / sqrt(((double)q0 + 0.5) / 512.0);  */
8854         q = int64_to_float64(q_int, s);
8855         q = float64_add(q, float64_half, s);
8856         q = float64_div(q, float64_512, s);
8857         q = float64_sqrt(q, s);
8858         q = float64_div(float64_one, q, s);
8859     } else {
8860         /* range 0.5 <= a < 1.0 */
8861
8862         /* a in units of 1/256 rounded down */
8863         /* q1 = (int)(a * 256.0); */
8864         q = float64_mul(float64_256, a, s);
8865         int64_t q_int = float64_to_int64_round_to_zero(q, s);
8866
8867         /* reciprocal root r */
8868         /* r = 1.0 /sqrt(((double)q1 + 0.5) / 256); */
8869         q = int64_to_float64(q_int, s);
8870         q = float64_add(q, float64_half, s);
8871         q = float64_div(q, float64_256, s);
8872         q = float64_sqrt(q, s);
8873         q = float64_div(float64_one, q, s);
8874     }
8875     /* r in units of 1/256 rounded to nearest */
8876     /* s = (int)(256.0 * r + 0.5); */
8877
8878     q = float64_mul(q, float64_256,s );
8879     q = float64_add(q, float64_half, s);
8880     q_int = float64_to_int64_round_to_zero(q, s);
8881
8882     /* return (double)s / 256.0;*/
8883     return float64_div(int64_to_float64(q_int, s), float64_256, s);
8884 }
8885
8886 float32 HELPER(rsqrte_f32)(float32 input, void *fpstp)
8887 {
8888     float_status *s = fpstp;
8889     float32 f32 = float32_squash_input_denormal(input, s);
8890     uint32_t val = float32_val(f32);
8891     uint32_t f32_sbit = 0x80000000 & val;
8892     int32_t f32_exp = extract32(val, 23, 8);
8893     uint32_t f32_frac = extract32(val, 0, 23);
8894     uint64_t f64_frac;
8895     uint64_t val64;
8896     int result_exp;
8897     float64 f64;
8898
8899     if (float32_is_any_nan(f32)) {
8900         float32 nan = f32;
8901         if (float32_is_signaling_nan(f32)) {
8902             float_raise(float_flag_invalid, s);
8903             nan = float32_maybe_silence_nan(f32);
8904         }
8905         if (s->default_nan_mode) {
8906             nan =  float32_default_nan;
8907         }
8908         return nan;
8909     } else if (float32_is_zero(f32)) {
8910         float_raise(float_flag_divbyzero, s);
8911         return float32_set_sign(float32_infinity, float32_is_neg(f32));
8912     } else if (float32_is_neg(f32)) {
8913         float_raise(float_flag_invalid, s);
8914         return float32_default_nan;
8915     } else if (float32_is_infinity(f32)) {
8916         return float32_zero;
8917     }
8918
8919     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
8920      * preserving the parity of the exponent.  */
8921
8922     f64_frac = ((uint64_t) f32_frac) << 29;
8923     if (f32_exp == 0) {
8924         while (extract64(f64_frac, 51, 1) == 0) {
8925             f64_frac = f64_frac << 1;
8926             f32_exp = f32_exp-1;
8927         }
8928         f64_frac = extract64(f64_frac, 0, 51) << 1;
8929     }
8930
8931     if (extract64(f32_exp, 0, 1) == 0) {
8932         f64 = make_float64(((uint64_t) f32_sbit) << 32
8933                            | (0x3feULL << 52)
8934                            | f64_frac);
8935     } else {
8936         f64 = make_float64(((uint64_t) f32_sbit) << 32
8937                            | (0x3fdULL << 52)
8938                            | f64_frac);
8939     }
8940
8941     result_exp = (380 - f32_exp) / 2;
8942
8943     f64 = recip_sqrt_estimate(f64, s);
8944
8945     val64 = float64_val(f64);
8946
8947     val = ((result_exp & 0xff) << 23)
8948         | ((val64 >> 29)  & 0x7fffff);
8949     return make_float32(val);
8950 }
8951
8952 float64 HELPER(rsqrte_f64)(float64 input, void *fpstp)
8953 {
8954     float_status *s = fpstp;
8955     float64 f64 = float64_squash_input_denormal(input, s);
8956     uint64_t val = float64_val(f64);
8957     uint64_t f64_sbit = 0x8000000000000000ULL & val;
8958     int64_t f64_exp = extract64(val, 52, 11);
8959     uint64_t f64_frac = extract64(val, 0, 52);
8960     int64_t result_exp;
8961     uint64_t result_frac;
8962
8963     if (float64_is_any_nan(f64)) {
8964         float64 nan = f64;
8965         if (float64_is_signaling_nan(f64)) {
8966             float_raise(float_flag_invalid, s);
8967             nan = float64_maybe_silence_nan(f64);
8968         }
8969         if (s->default_nan_mode) {
8970             nan =  float64_default_nan;
8971         }
8972         return nan;
8973     } else if (float64_is_zero(f64)) {
8974         float_raise(float_flag_divbyzero, s);
8975         return float64_set_sign(float64_infinity, float64_is_neg(f64));
8976     } else if (float64_is_neg(f64)) {
8977         float_raise(float_flag_invalid, s);
8978         return float64_default_nan;
8979     } else if (float64_is_infinity(f64)) {
8980         return float64_zero;
8981     }
8982
8983     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
8984      * preserving the parity of the exponent.  */
8985
8986     if (f64_exp == 0) {
8987         while (extract64(f64_frac, 51, 1) == 0) {
8988             f64_frac = f64_frac << 1;
8989             f64_exp = f64_exp - 1;
8990         }
8991         f64_frac = extract64(f64_frac, 0, 51) << 1;
8992     }
8993
8994     if (extract64(f64_exp, 0, 1) == 0) {
8995         f64 = make_float64(f64_sbit
8996                            | (0x3feULL << 52)
8997                            | f64_frac);
8998     } else {
8999         f64 = make_float64(f64_sbit
9000                            | (0x3fdULL << 52)
9001                            | f64_frac);
9002     }
9003
9004     result_exp = (3068 - f64_exp) / 2;
9005
9006     f64 = recip_sqrt_estimate(f64, s);
9007
9008     result_frac = extract64(float64_val(f64), 0, 52);
9009
9010     return make_float64(f64_sbit |
9011                         ((result_exp & 0x7ff) << 52) |
9012                         result_frac);
9013 }
9014
9015 uint32_t HELPER(recpe_u32)(uint32_t a, void *fpstp)
9016 {
9017     float_status *s = fpstp;
9018     float64 f64;
9019
9020     if ((a & 0x80000000) == 0) {
9021         return 0xffffffff;
9022     }
9023
9024     f64 = make_float64((0x3feULL << 52)
9025                        | ((int64_t)(a & 0x7fffffff) << 21));
9026
9027     f64 = recip_estimate(f64, s);
9028
9029     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
9030 }
9031
9032 uint32_t HELPER(rsqrte_u32)(uint32_t a, void *fpstp)
9033 {
9034     float_status *fpst = fpstp;
9035     float64 f64;
9036
9037     if ((a & 0xc0000000) == 0) {
9038         return 0xffffffff;
9039     }
9040
9041     if (a & 0x80000000) {
9042         f64 = make_float64((0x3feULL << 52)
9043                            | ((uint64_t)(a & 0x7fffffff) << 21));
9044     } else { /* bits 31-30 == '01' */
9045         f64 = make_float64((0x3fdULL << 52)
9046                            | ((uint64_t)(a & 0x3fffffff) << 22));
9047     }
9048
9049     f64 = recip_sqrt_estimate(f64, fpst);
9050
9051     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
9052 }
9053
9054 /* VFPv4 fused multiply-accumulate */
9055 float32 VFP_HELPER(muladd, s)(float32 a, float32 b, float32 c, void *fpstp)
9056 {
9057     float_status *fpst = fpstp;
9058     return float32_muladd(a, b, c, 0, fpst);
9059 }
9060
9061 float64 VFP_HELPER(muladd, d)(float64 a, float64 b, float64 c, void *fpstp)
9062 {
9063     float_status *fpst = fpstp;
9064     return float64_muladd(a, b, c, 0, fpst);
9065 }
9066
9067 /* ARMv8 round to integral */
9068 float32 HELPER(rints_exact)(float32 x, void *fp_status)
9069 {
9070     return float32_round_to_int(x, fp_status);
9071 }
9072
9073 float64 HELPER(rintd_exact)(float64 x, void *fp_status)
9074 {
9075     return float64_round_to_int(x, fp_status);
9076 }
9077
9078 float32 HELPER(rints)(float32 x, void *fp_status)
9079 {
9080     int old_flags = get_float_exception_flags(fp_status), new_flags;
9081     float32 ret;
9082
9083     ret = float32_round_to_int(x, fp_status);
9084
9085     /* Suppress any inexact exceptions the conversion produced */
9086     if (!(old_flags & float_flag_inexact)) {
9087         new_flags = get_float_exception_flags(fp_status);
9088         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
9089     }
9090
9091     return ret;
9092 }
9093
9094 float64 HELPER(rintd)(float64 x, void *fp_status)
9095 {
9096     int old_flags = get_float_exception_flags(fp_status), new_flags;
9097     float64 ret;
9098
9099     ret = float64_round_to_int(x, fp_status);
9100
9101     new_flags = get_float_exception_flags(fp_status);
9102
9103     /* Suppress any inexact exceptions the conversion produced */
9104     if (!(old_flags & float_flag_inexact)) {
9105         new_flags = get_float_exception_flags(fp_status);
9106         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
9107     }
9108
9109     return ret;
9110 }
9111
9112 /* Convert ARM rounding mode to softfloat */
9113 int arm_rmode_to_sf(int rmode)
9114 {
9115     switch (rmode) {
9116     case FPROUNDING_TIEAWAY:
9117         rmode = float_round_ties_away;
9118         break;
9119     case FPROUNDING_ODD:
9120         /* FIXME: add support for TIEAWAY and ODD */
9121         qemu_log_mask(LOG_UNIMP, "arm: unimplemented rounding mode: %d\n",
9122                       rmode);
9123     case FPROUNDING_TIEEVEN:
9124     default:
9125         rmode = float_round_nearest_even;
9126         break;
9127     case FPROUNDING_POSINF:
9128         rmode = float_round_up;
9129         break;
9130     case FPROUNDING_NEGINF:
9131         rmode = float_round_down;
9132         break;
9133     case FPROUNDING_ZERO:
9134         rmode = float_round_to_zero;
9135         break;
9136     }
9137     return rmode;
9138 }
9139
9140 /* CRC helpers.
9141  * The upper bytes of val (above the number specified by 'bytes') must have
9142  * been zeroed out by the caller.
9143  */
9144 uint32_t HELPER(crc32)(uint32_t acc, uint32_t val, uint32_t bytes)
9145 {
9146     uint8_t buf[4];
9147
9148     stl_le_p(buf, val);
9149
9150     /* zlib crc32 converts the accumulator and output to one's complement.  */
9151     return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
9152 }
9153
9154 uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
9155 {
9156     uint8_t buf[4];
9157
9158     stl_le_p(buf, val);
9159
9160     /* Linux crc32c converts the output to one's complement.  */
9161     return crc32c(acc, buf, bytes) ^ 0xffffffff;
9162 }