target-arm/helper.c

   1 #include "cpu.h"
   2 #include "internals.h"
   3 #include "exec/gdbstub.h"
   4 #include "exec/helper-proto.h"
   5 #include "qemu/host-utils.h"
   6 #include "sysemu/arch_init.h"
   7 #include "sysemu/sysemu.h"
   8 #include "qemu/bitops.h"
   9 #include "qemu/crc32c.h"
  10 #include "exec/cpu_ldst.h"
  11 #include "arm_ldst.h"
  12 #include <zlib.h> /* For crc32 */
  13
  14 #ifndef CONFIG_USER_ONLY
  15 static inline int get_phys_addr(CPUARMState *env, target_ulong address,
  16                                 int access_type, ARMMMUIdx mmu_idx,
  17                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
  18                                 target_ulong *page_size);
  19
  20 /* Definitions for the PMCCNTR and PMCR registers */
  21 #define PMCRD   0x8
  22 #define PMCRC   0x4
  23 #define PMCRE   0x1
  24 #endif
  25
  26 static int vfp_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  27 {
  28     int nregs;
  29
  30     /* VFP data registers are always little-endian.  */
  31     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  32     if (reg < nregs) {
  33         stfq_le_p(buf, env->vfp.regs[reg]);
  34         return 8;
  35     }
  36     if (arm_feature(env, ARM_FEATURE_NEON)) {
  37         /* Aliases for Q regs.  */
  38         nregs += 16;
  39         if (reg < nregs) {
  40             stfq_le_p(buf, env->vfp.regs[(reg - 32) * 2]);
  41             stfq_le_p(buf + 8, env->vfp.regs[(reg - 32) * 2 + 1]);
  42             return 16;
  43         }
  44     }
  45     switch (reg - nregs) {
  46     case 0: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSID]); return 4;
  47     case 1: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSCR]); return 4;
  48     case 2: stl_p(buf, env->vfp.xregs[ARM_VFP_FPEXC]); return 4;
  49     }
  50     return 0;
  51 }
  52
  53 static int vfp_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
  54 {
  55     int nregs;
  56
  57     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  58     if (reg < nregs) {
  59         env->vfp.regs[reg] = ldfq_le_p(buf);
  60         return 8;
  61     }
  62     if (arm_feature(env, ARM_FEATURE_NEON)) {
  63         nregs += 16;
  64         if (reg < nregs) {
  65             env->vfp.regs[(reg - 32) * 2] = ldfq_le_p(buf);
  66             env->vfp.regs[(reg - 32) * 2 + 1] = ldfq_le_p(buf + 8);
  67             return 16;
  68         }
  69     }
  70     switch (reg - nregs) {
  71     case 0: env->vfp.xregs[ARM_VFP_FPSID] = ldl_p(buf); return 4;
  72     case 1: env->vfp.xregs[ARM_VFP_FPSCR] = ldl_p(buf); return 4;
  73     case 2: env->vfp.xregs[ARM_VFP_FPEXC] = ldl_p(buf) & (1 << 30); return 4;
  74     }
  75     return 0;
  76 }
  77
  78 static int aarch64_fpu_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  79 {
  80     switch (reg) {
  81     case 0 ... 31:
  82         /* 128 bit FP register */
  83         stfq_le_p(buf, env->vfp.regs[reg * 2]);
  84         stfq_le_p(buf + 8, env->vfp.regs[reg * 2 + 1]);
  85         return 16;
  86     case 32:
  87         /* FPSR */
  88         stl_p(buf, vfp_get_fpsr(env));
  89         return 4;
  90     case 33:
  91         /* FPCR */
  92         stl_p(buf, vfp_get_fpcr(env));
  93         return 4;
  94     default:
  95         return 0;
  96     }
  97 }
  98
  99 static int aarch64_fpu_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
 100 {
 101     switch (reg) {
 102     case 0 ... 31:
 103         /* 128 bit FP register */
 104         env->vfp.regs[reg * 2] = ldfq_le_p(buf);
 105         env->vfp.regs[reg * 2 + 1] = ldfq_le_p(buf + 8);
 106         return 16;
 107     case 32:
 108         /* FPSR */
 109         vfp_set_fpsr(env, ldl_p(buf));
 110         return 4;
 111     case 33:
 112         /* FPCR */
 113         vfp_set_fpcr(env, ldl_p(buf));
 114         return 4;
 115     default:
 116         return 0;
 117     }
 118 }
 119
 120 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
 121 {
 122     assert(ri->fieldoffset);
 123     if (cpreg_field_is_64bit(ri)) {
 124         return CPREG_FIELD64(env, ri);
 125     } else {
 126         return CPREG_FIELD32(env, ri);
 127     }
 128 }
 129
 130 static void raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
 131                       uint64_t value)
 132 {
 133     assert(ri->fieldoffset);
 134     if (cpreg_field_is_64bit(ri)) {
 135         CPREG_FIELD64(env, ri) = value;
 136     } else {
 137         CPREG_FIELD32(env, ri) = value;
 138     }
 139 }
 140
 141 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 142 {
 143     return (char *)env + ri->fieldoffset;
 144 }
 145
 146 static uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 147 {
 148     /* Raw read of a coprocessor register (as needed for migration, etc). */
 149     if (ri->type & ARM_CP_CONST) {
 150         return ri->resetvalue;
 151     } else if (ri->raw_readfn) {
 152         return ri->raw_readfn(env, ri);
 153     } else if (ri->readfn) {
 154         return ri->readfn(env, ri);
 155     } else {
 156         return raw_read(env, ri);
 157     }
 158 }
 159
 160 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
 161                              uint64_t v)
 162 {
 163     /* Raw write of a coprocessor register (as needed for migration, etc).
 164      * Note that constant registers are treated as write-ignored; the
 165      * caller should check for success by whether a readback gives the
 166      * value written.
 167      */
 168     if (ri->type & ARM_CP_CONST) {
 169         return;
 170     } else if (ri->raw_writefn) {
 171         ri->raw_writefn(env, ri, v);
 172     } else if (ri->writefn) {
 173         ri->writefn(env, ri, v);
 174     } else {
 175         raw_write(env, ri, v);
 176     }
 177 }
 178
 179 static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
 180 {
 181    /* Return true if the regdef would cause an assertion if you called
 182     * read_raw_cp_reg() or write_raw_cp_reg() on it (ie if it is a
 183     * program bug for it not to have the NO_RAW flag).
 184     * NB that returning false here doesn't necessarily mean that calling
 185     * read/write_raw_cp_reg() is safe, because we can't distinguish "has
 186     * read/write access functions which are safe for raw use" from "has
 187     * read/write access functions which have side effects but has forgotten
 188     * to provide raw access functions".
 189     * The tests here line up with the conditions in read/write_raw_cp_reg()
 190     * and assertions in raw_read()/raw_write().
 191     */
 192     if ((ri->type & ARM_CP_CONST) ||
 193         ri->fieldoffset ||
 194         ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
 195         return false;
 196     }
 197     return true;
 198 }
 199
 200 bool write_cpustate_to_list(ARMCPU *cpu)
 201 {
 202     /* Write the coprocessor state from cpu->env to the (index,value) list. */
 203     int i;
 204     bool ok = true;
 205
 206     for (i = 0; i < cpu->cpreg_array_len; i++) {
 207         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 208         const ARMCPRegInfo *ri;
 209
 210         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 211         if (!ri) {
 212             ok = false;
 213             continue;
 214         }
 215         if (ri->type & ARM_CP_NO_RAW) {
 216             continue;
 217         }
 218         cpu->cpreg_values[i] = read_raw_cp_reg(&cpu->env, ri);
 219     }
 220     return ok;
 221 }
 222
 223 bool write_list_to_cpustate(ARMCPU *cpu)
 224 {
 225     int i;
 226     bool ok = true;
 227
 228     for (i = 0; i < cpu->cpreg_array_len; i++) {
 229         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 230         uint64_t v = cpu->cpreg_values[i];
 231         const ARMCPRegInfo *ri;
 232
 233         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 234         if (!ri) {
 235             ok = false;
 236             continue;
 237         }
 238         if (ri->type & ARM_CP_NO_RAW) {
 239             continue;
 240         }
 241         /* Write value and confirm it reads back as written
 242          * (to catch read-only registers and partially read-only
 243          * registers where the incoming migration value doesn't match)
 244          */
 245         write_raw_cp_reg(&cpu->env, ri, v);
 246         if (read_raw_cp_reg(&cpu->env, ri) != v) {
 247             ok = false;
 248         }
 249     }
 250     return ok;
 251 }
 252
 253 static void add_cpreg_to_list(gpointer key, gpointer opaque)
 254 {
 255     ARMCPU *cpu = opaque;
 256     uint64_t regidx;
 257     const ARMCPRegInfo *ri;
 258
 259     regidx = *(uint32_t *)key;
 260     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 261
 262     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 263         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
 264         /* The value array need not be initialized at this point */
 265         cpu->cpreg_array_len++;
 266     }
 267 }
 268
 269 static void count_cpreg(gpointer key, gpointer opaque)
 270 {
 271     ARMCPU *cpu = opaque;
 272     uint64_t regidx;
 273     const ARMCPRegInfo *ri;
 274
 275     regidx = *(uint32_t *)key;
 276     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 277
 278     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 279         cpu->cpreg_array_len++;
 280     }
 281 }
 282
 283 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
 284 {
 285     uint64_t aidx = cpreg_to_kvm_id(*(uint32_t *)a);
 286     uint64_t bidx = cpreg_to_kvm_id(*(uint32_t *)b);
 287
 288     if (aidx > bidx) {
 289         return 1;
 290     }
 291     if (aidx < bidx) {
 292         return -1;
 293     }
 294     return 0;
 295 }
 296
 297 static void cpreg_make_keylist(gpointer key, gpointer value, gpointer udata)
 298 {
 299     GList **plist = udata;
 300
 301     *plist = g_list_prepend(*plist, key);
 302 }
 303
 304 void init_cpreg_list(ARMCPU *cpu)
 305 {
 306     /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
 307      * Note that we require cpreg_tuples[] to be sorted by key ID.
 308      */
 309     GList *keys = NULL;
 310     int arraylen;
 311
 312     g_hash_table_foreach(cpu->cp_regs, cpreg_make_keylist, &keys);
 313
 314     keys = g_list_sort(keys, cpreg_key_compare);
 315
 316     cpu->cpreg_array_len = 0;
 317
 318     g_list_foreach(keys, count_cpreg, cpu);
 319
 320     arraylen = cpu->cpreg_array_len;
 321     cpu->cpreg_indexes = g_new(uint64_t, arraylen);
 322     cpu->cpreg_values = g_new(uint64_t, arraylen);
 323     cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
 324     cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
 325     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
 326     cpu->cpreg_array_len = 0;
 327
 328     g_list_foreach(keys, add_cpreg_to_list, cpu);
 329
 330     assert(cpu->cpreg_array_len == arraylen);
 331
 332     g_list_free(keys);
 333 }
 334
 335 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 336 {
 337     ARMCPU *cpu = arm_env_get_cpu(env);
 338
 339     raw_write(env, ri, value);
 340     tlb_flush(CPU(cpu), 1); /* Flush TLB as domain not tracked in TLB */
 341 }
 342
 343 static void fcse_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 344 {
 345     ARMCPU *cpu = arm_env_get_cpu(env);
 346
 347     if (raw_read(env, ri) != value) {
 348         /* Unlike real hardware the qemu TLB uses virtual addresses,
 349          * not modified virtual addresses, so this causes a TLB flush.
 350          */
 351         tlb_flush(CPU(cpu), 1);
 352         raw_write(env, ri, value);
 353     }
 354 }
 355
 356 static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 357                              uint64_t value)
 358 {
 359     ARMCPU *cpu = arm_env_get_cpu(env);
 360
 361     if (raw_read(env, ri) != value && !arm_feature(env, ARM_FEATURE_MPU)
 362         && !extended_addresses_enabled(env)) {
 363         /* For VMSA (when not using the LPAE long descriptor page table
 364          * format) this register includes the ASID, so do a TLB flush.
 365          * For PMSA it is purely a process ID and no action is needed.
 366          */
 367         tlb_flush(CPU(cpu), 1);
 368     }
 369     raw_write(env, ri, value);
 370 }
 371
 372 static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
 373                           uint64_t value)
 374 {
 375     /* Invalidate all (TLBIALL) */
 376     ARMCPU *cpu = arm_env_get_cpu(env);
 377
 378     tlb_flush(CPU(cpu), 1);
 379 }
 380
 381 static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri,
 382                           uint64_t value)
 383 {
 384     /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */
 385     ARMCPU *cpu = arm_env_get_cpu(env);
 386
 387     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 388 }
 389
 390 static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri,
 391                            uint64_t value)
 392 {
 393     /* Invalidate by ASID (TLBIASID) */
 394     ARMCPU *cpu = arm_env_get_cpu(env);
 395
 396     tlb_flush(CPU(cpu), value == 0);
 397 }
 398
 399 static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
 400                            uint64_t value)
 401 {
 402     /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */
 403     ARMCPU *cpu = arm_env_get_cpu(env);
 404
 405     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 406 }
 407
 408 /* IS variants of TLB operations must affect all cores */
 409 static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 410                              uint64_t value)
 411 {
 412     CPUState *other_cs;
 413
 414     CPU_FOREACH(other_cs) {
 415         tlb_flush(other_cs, 1);
 416     }
 417 }
 418
 419 static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 420                              uint64_t value)
 421 {
 422     CPUState *other_cs;
 423
 424     CPU_FOREACH(other_cs) {
 425         tlb_flush(other_cs, value == 0);
 426     }
 427 }
 428
 429 static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 430                              uint64_t value)
 431 {
 432     CPUState *other_cs;
 433
 434     CPU_FOREACH(other_cs) {
 435         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 436     }
 437 }
 438
 439 static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 440                              uint64_t value)
 441 {
 442     CPUState *other_cs;
 443
 444     CPU_FOREACH(other_cs) {
 445         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 446     }
 447 }
 448
 449 static const ARMCPRegInfo cp_reginfo[] = {
 450     /* Define the secure and non-secure FCSE identifier CP registers
 451      * separately because there is no secure bank in V8 (no _EL3).  This allows
 452      * the secure register to be properly reset and migrated. There is also no
 453      * v8 EL1 version of the register so the non-secure instance stands alone.
 454      */
 455     { .name = "FCSEIDR(NS)",
 456       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 457       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 458       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
 459       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 460     { .name = "FCSEIDR(S)",
 461       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 462       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 463       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
 464       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 465     /* Define the secure and non-secure context identifier CP registers
 466      * separately because there is no secure bank in V8 (no _EL3).  This allows
 467      * the secure register to be properly reset and migrated.  In the
 468      * non-secure case, the 32-bit register will have reset and migration
 469      * disabled during registration as it is handled by the 64-bit instance.
 470      */
 471     { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
 472       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 473       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 474       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
 475       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 476     { .name = "CONTEXTIDR(S)", .state = ARM_CP_STATE_AA32,
 477       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 478       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 479       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
 480       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 481     REGINFO_SENTINEL
 482 };
 483
 484 static const ARMCPRegInfo not_v8_cp_reginfo[] = {
 485     /* NB: Some of these registers exist in v8 but with more precise
 486      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
 487      */
 488     /* MMU Domain access control / MPU write buffer control */
 489     { .name = "DACR",
 490       .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
 491       .access = PL1_RW, .resetvalue = 0,
 492       .writefn = dacr_write, .raw_writefn = raw_write,
 493       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
 494                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
 495     /* ARMv7 allocates a range of implementation defined TLB LOCKDOWN regs.
 496      * For v6 and v5, these mappings are overly broad.
 497      */
 498     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 0,
 499       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 500     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 1,
 501       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 502     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 4,
 503       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 504     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 8,
 505       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 506     /* Cache maintenance ops; some of this space may be overridden later. */
 507     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
 508       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
 509       .type = ARM_CP_NOP | ARM_CP_OVERRIDE },
 510     REGINFO_SENTINEL
 511 };
 512
 513 static const ARMCPRegInfo not_v6_cp_reginfo[] = {
 514     /* Not all pre-v6 cores implemented this WFI, so this is slightly
 515      * over-broad.
 516      */
 517     { .name = "WFI_v5", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = 2,
 518       .access = PL1_W, .type = ARM_CP_WFI },
 519     REGINFO_SENTINEL
 520 };
 521
 522 static const ARMCPRegInfo not_v7_cp_reginfo[] = {
 523     /* Standard v6 WFI (also used in some pre-v6 cores); not in v7 (which
 524      * is UNPREDICTABLE; we choose to NOP as most implementations do).
 525      */
 526     { .name = "WFI_v6", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 527       .access = PL1_W, .type = ARM_CP_WFI },
 528     /* L1 cache lockdown. Not architectural in v6 and earlier but in practice
 529      * implemented in 926, 946, 1026, 1136, 1176 and 11MPCore. StrongARM and
 530      * OMAPCP will override this space.
 531      */
 532     { .name = "DLOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 0,
 533       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_data),
 534       .resetvalue = 0 },
 535     { .name = "ILOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 1,
 536       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_insn),
 537       .resetvalue = 0 },
 538     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
 539     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
 540       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
 541       .resetvalue = 0 },
 542     /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
 543      * implementing it as RAZ means the "debug architecture version" bits
 544      * will read as a reserved value, which should cause Linux to not try
 545      * to use the debug hardware.
 546      */
 547     { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
 548       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
 549     /* MMU TLB control. Note that the wildcarding means we cover not just
 550      * the unified TLB ops but also the dside/iside/inner-shareable variants.
 551      */
 552     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
 553       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
 554       .type = ARM_CP_NO_RAW },
 555     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
 556       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
 557       .type = ARM_CP_NO_RAW },
 558     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
 559       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
 560       .type = ARM_CP_NO_RAW },
 561     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
 562       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
 563       .type = ARM_CP_NO_RAW },
 564     { .name = "PRRR", .cp = 15, .crn = 10, .crm = 2,
 565       .opc1 = 0, .opc2 = 0, .access = PL1_RW, .type = ARM_CP_NOP },
 566     { .name = "NMRR", .cp = 15, .crn = 10, .crm = 2,
 567       .opc1 = 0, .opc2 = 1, .access = PL1_RW, .type = ARM_CP_NOP },
 568     REGINFO_SENTINEL
 569 };
 570
 571 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 572                         uint64_t value)
 573 {
 574     uint32_t mask = 0;
 575
 576     /* In ARMv8 most bits of CPACR_EL1 are RES0. */
 577     if (!arm_feature(env, ARM_FEATURE_V8)) {
 578         /* ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
 579          * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
 580          * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
 581          */
 582         if (arm_feature(env, ARM_FEATURE_VFP)) {
 583             /* VFP coprocessor: cp10 & cp11 [23:20] */
 584             mask |= (1 << 31) | (1 << 30) | (0xf << 20);
 585
 586             if (!arm_feature(env, ARM_FEATURE_NEON)) {
 587                 /* ASEDIS [31] bit is RAO/WI */
 588                 value |= (1 << 31);
 589             }
 590
 591             /* VFPv3 and upwards with NEON implement 32 double precision
 592              * registers (D0-D31).
 593              */
 594             if (!arm_feature(env, ARM_FEATURE_NEON) ||
 595                     !arm_feature(env, ARM_FEATURE_VFP3)) {
 596                 /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
 597                 value |= (1 << 30);
 598             }
 599         }
 600         value &= mask;
 601     }
 602     env->cp15.cpacr_el1 = value;
 603 }
 604
 605 static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri)
 606 {
 607     if (arm_feature(env, ARM_FEATURE_V8)) {
 608         /* Check if CPACR accesses are to be trapped to EL2 */
 609         if (arm_current_el(env) == 1 &&
 610             (env->cp15.cptr_el[2] & CPTR_TCPAC) && !arm_is_secure(env)) {
 611             return CP_ACCESS_TRAP_EL2;
 612         /* Check if CPACR accesses are to be trapped to EL3 */
 613         } else if (arm_current_el(env) < 3 &&
 614                    (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
 615             return CP_ACCESS_TRAP_EL3;
 616         }
 617     }
 618
 619     return CP_ACCESS_OK;
 620 }
 621
 622 static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri)
 623 {
 624     /* Check if CPTR accesses are set to trap to EL3 */
 625     if (arm_current_el(env) == 2 && (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
 626         return CP_ACCESS_TRAP_EL3;
 627     }
 628
 629     return CP_ACCESS_OK;
 630 }
 631
 632 static const ARMCPRegInfo v6_cp_reginfo[] = {
 633     /* prefetch by MVA in v6, NOP in v7 */
 634     { .name = "MVA_prefetch",
 635       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
 636       .access = PL1_W, .type = ARM_CP_NOP },
 637     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
 638       .access = PL0_W, .type = ARM_CP_NOP },
 639     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
 640       .access = PL0_W, .type = ARM_CP_NOP },
 641     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
 642       .access = PL0_W, .type = ARM_CP_NOP },
 643     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
 644       .access = PL1_RW,
 645       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
 646                              offsetof(CPUARMState, cp15.ifar_ns) },
 647       .resetvalue = 0, },
 648     /* Watchpoint Fault Address Register : should actually only be present
 649      * for 1136, 1176, 11MPCore.
 650      */
 651     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
 652       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
 653     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
 654       .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2, .accessfn = cpacr_access,
 655       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.cpacr_el1),
 656       .resetvalue = 0, .writefn = cpacr_write },
 657     REGINFO_SENTINEL
 658 };
 659
 660 static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri)
 661 {
 662     /* Performance monitor registers user accessibility is controlled
 663      * by PMUSERENR.
 664      */
 665     if (arm_current_el(env) == 0 && !env->cp15.c9_pmuserenr) {
 666         return CP_ACCESS_TRAP;
 667     }
 668     return CP_ACCESS_OK;
 669 }
 670
 671 #ifndef CONFIG_USER_ONLY
 672
 673 static inline bool arm_ccnt_enabled(CPUARMState *env)
 674 {
 675     /* This does not support checking PMCCFILTR_EL0 register */
 676
 677     if (!(env->cp15.c9_pmcr & PMCRE)) {
 678         return false;
 679     }
 680
 681     return true;
 682 }
 683
 684 void pmccntr_sync(CPUARMState *env)
 685 {
 686     uint64_t temp_ticks;
 687
 688     temp_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 689                           get_ticks_per_sec(), 1000000);
 690
 691     if (env->cp15.c9_pmcr & PMCRD) {
 692         /* Increment once every 64 processor clock cycles */
 693         temp_ticks /= 64;
 694     }
 695
 696     if (arm_ccnt_enabled(env)) {
 697         env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
 698     }
 699 }
 700
 701 static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 702                        uint64_t value)
 703 {
 704     pmccntr_sync(env);
 705
 706     if (value & PMCRC) {
 707         /* The counter has been reset */
 708         env->cp15.c15_ccnt = 0;
 709     }
 710
 711     /* only the DP, X, D and E bits are writable */
 712     env->cp15.c9_pmcr &= ~0x39;
 713     env->cp15.c9_pmcr |= (value & 0x39);
 714
 715     pmccntr_sync(env);
 716 }
 717
 718 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 719 {
 720     uint64_t total_ticks;
 721
 722     if (!arm_ccnt_enabled(env)) {
 723         /* Counter is disabled, do not change value */
 724         return env->cp15.c15_ccnt;
 725     }
 726
 727     total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 728                            get_ticks_per_sec(), 1000000);
 729
 730     if (env->cp15.c9_pmcr & PMCRD) {
 731         /* Increment once every 64 processor clock cycles */
 732         total_ticks /= 64;
 733     }
 734     return total_ticks - env->cp15.c15_ccnt;
 735 }
 736
 737 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 738                         uint64_t value)
 739 {
 740     uint64_t total_ticks;
 741
 742     if (!arm_ccnt_enabled(env)) {
 743         /* Counter is disabled, set the absolute value */
 744         env->cp15.c15_ccnt = value;
 745         return;
 746     }
 747
 748     total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 749                            get_ticks_per_sec(), 1000000);
 750
 751     if (env->cp15.c9_pmcr & PMCRD) {
 752         /* Increment once every 64 processor clock cycles */
 753         total_ticks /= 64;
 754     }
 755     env->cp15.c15_ccnt = total_ticks - value;
 756 }
 757
 758 static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
 759                             uint64_t value)
 760 {
 761     uint64_t cur_val = pmccntr_read(env, NULL);
 762
 763     pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
 764 }
 765
 766 #else /* CONFIG_USER_ONLY */
 767
 768 void pmccntr_sync(CPUARMState *env)
 769 {
 770 }
 771
 772 #endif
 773
 774 static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 775                             uint64_t value)
 776 {
 777     pmccntr_sync(env);
 778     env->cp15.pmccfiltr_el0 = value & 0x7E000000;
 779     pmccntr_sync(env);
 780 }
 781
 782 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 783                             uint64_t value)
 784 {
 785     value &= (1 << 31);
 786     env->cp15.c9_pmcnten |= value;
 787 }
 788
 789 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 790                              uint64_t value)
 791 {
 792     value &= (1 << 31);
 793     env->cp15.c9_pmcnten &= ~value;
 794 }
 795
 796 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 797                          uint64_t value)
 798 {
 799     env->cp15.c9_pmovsr &= ~value;
 800 }
 801
 802 static void pmxevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
 803                              uint64_t value)
 804 {
 805     env->cp15.c9_pmxevtyper = value & 0xff;
 806 }
 807
 808 static void pmuserenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 809                             uint64_t value)
 810 {
 811     env->cp15.c9_pmuserenr = value & 1;
 812 }
 813
 814 static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 815                              uint64_t value)
 816 {
 817     /* We have no event counters so only the C bit can be changed */
 818     value &= (1 << 31);
 819     env->cp15.c9_pminten |= value;
 820 }
 821
 822 static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 823                              uint64_t value)
 824 {
 825     value &= (1 << 31);
 826     env->cp15.c9_pminten &= ~value;
 827 }
 828
 829 static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
 830                        uint64_t value)
 831 {
 832     /* Note that even though the AArch64 view of this register has bits
 833      * [10:0] all RES0 we can only mask the bottom 5, to comply with the
 834      * architectural requirements for bits which are RES0 only in some
 835      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
 836      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
 837      */
 838     raw_write(env, ri, value & ~0x1FULL);
 839 }
 840
 841 static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 842 {
 843     /* We only mask off bits that are RES0 both for AArch64 and AArch32.
 844      * For bits that vary between AArch32/64, code needs to check the
 845      * current execution mode before directly using the feature bit.
 846      */
 847     uint32_t valid_mask = SCR_AARCH64_MASK | SCR_AARCH32_MASK;
 848
 849     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 850         valid_mask &= ~SCR_HCE;
 851
 852         /* On ARMv7, SMD (or SCD as it is called in v7) is only
 853          * supported if EL2 exists. The bit is UNK/SBZP when
 854          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
 855          * when EL2 is unavailable.
 856          * On ARMv8, this bit is always available.
 857          */
 858         if (arm_feature(env, ARM_FEATURE_V7) &&
 859             !arm_feature(env, ARM_FEATURE_V8)) {
 860             valid_mask &= ~SCR_SMD;
 861         }
 862     }
 863
 864     /* Clear all-context RES0 bits.  */
 865     value &= valid_mask;
 866     raw_write(env, ri, value);
 867 }
 868
 869 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 870 {
 871     ARMCPU *cpu = arm_env_get_cpu(env);
 872
 873     /* Acquire the CSSELR index from the bank corresponding to the CCSIDR
 874      * bank
 875      */
 876     uint32_t index = A32_BANKED_REG_GET(env, csselr,
 877                                         ri->secure & ARM_CP_SECSTATE_S);
 878
 879     return cpu->ccsidr[index];
 880 }
 881
 882 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 883                          uint64_t value)
 884 {
 885     raw_write(env, ri, value & 0xf);
 886 }
 887
 888 static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 889 {
 890     CPUState *cs = ENV_GET_CPU(env);
 891     uint64_t ret = 0;
 892
 893     if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
 894         ret |= CPSR_I;
 895     }
 896     if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
 897         ret |= CPSR_F;
 898     }
 899     /* External aborts are not possible in QEMU so A bit is always clear */
 900     return ret;
 901 }
 902
 903 static const ARMCPRegInfo v7_cp_reginfo[] = {
 904     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
 905     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 906       .access = PL1_W, .type = ARM_CP_NOP },
 907     /* Performance monitors are implementation defined in v7,
 908      * but with an ARM recommended set of registers, which we
 909      * follow (although we don't actually implement any counters)
 910      *
 911      * Performance registers fall into three categories:
 912      *  (a) always UNDEF in PL0, RW in PL1 (PMINTENSET, PMINTENCLR)
 913      *  (b) RO in PL0 (ie UNDEF on write), RW in PL1 (PMUSERENR)
 914      *  (c) UNDEF in PL0 if PMUSERENR.EN==0, otherwise accessible (all others)
 915      * For the cases controlled by PMUSERENR we must set .access to PL0_RW
 916      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
 917      */
 918     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
 919       .access = PL0_RW, .type = ARM_CP_ALIAS,
 920       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
 921       .writefn = pmcntenset_write,
 922       .accessfn = pmreg_access,
 923       .raw_writefn = raw_write },
 924     { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64,
 925       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
 926       .access = PL0_RW, .accessfn = pmreg_access,
 927       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
 928       .writefn = pmcntenset_write, .raw_writefn = raw_write },
 929     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
 930       .access = PL0_RW,
 931       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
 932       .accessfn = pmreg_access,
 933       .writefn = pmcntenclr_write,
 934       .type = ARM_CP_ALIAS },
 935     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
 936       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
 937       .access = PL0_RW, .accessfn = pmreg_access,
 938       .type = ARM_CP_ALIAS,
 939       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
 940       .writefn = pmcntenclr_write },
 941     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
 942       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
 943       .accessfn = pmreg_access,
 944       .writefn = pmovsr_write,
 945       .raw_writefn = raw_write },
 946     /* Unimplemented so WI. */
 947     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
 948       .access = PL0_W, .accessfn = pmreg_access, .type = ARM_CP_NOP },
 949     /* Since we don't implement any events, writing to PMSELR is UNPREDICTABLE.
 950      * We choose to RAZ/WI.
 951      */
 952     { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5,
 953       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
 954       .accessfn = pmreg_access },
 955 #ifndef CONFIG_USER_ONLY
 956     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
 957       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_IO,
 958       .readfn = pmccntr_read, .writefn = pmccntr_write32,
 959       .accessfn = pmreg_access },
 960     { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
 961       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
 962       .access = PL0_RW, .accessfn = pmreg_access,
 963       .type = ARM_CP_IO,
 964       .readfn = pmccntr_read, .writefn = pmccntr_write, },
 965 #endif
 966     { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
 967       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
 968       .writefn = pmccfiltr_write,
 969       .access = PL0_RW, .accessfn = pmreg_access,
 970       .type = ARM_CP_IO,
 971       .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
 972       .resetvalue = 0, },
 973     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
 974       .access = PL0_RW,
 975       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmxevtyper),
 976       .accessfn = pmreg_access, .writefn = pmxevtyper_write,
 977       .raw_writefn = raw_write },
 978     /* Unimplemented, RAZ/WI. */
 979     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
 980       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
 981       .accessfn = pmreg_access },
 982     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
 983       .access = PL0_R | PL1_RW,
 984       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
 985       .resetvalue = 0,
 986       .writefn = pmuserenr_write, .raw_writefn = raw_write },
 987     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
 988       .access = PL1_RW,
 989       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
 990       .resetvalue = 0,
 991       .writefn = pmintenset_write, .raw_writefn = raw_write },
 992     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
 993       .access = PL1_RW, .type = ARM_CP_ALIAS,
 994       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
 995       .resetvalue = 0, .writefn = pmintenclr_write, },
 996     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
 997       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
 998       .access = PL1_RW, .writefn = vbar_write,
 999       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
1000                              offsetof(CPUARMState, cp15.vbar_ns) },
1001       .resetvalue = 0 },
1002     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
1003       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
1004       .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_RAW },
1005     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
1006       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
1007       .access = PL1_RW, .writefn = csselr_write, .resetvalue = 0,
1008       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
1009                              offsetof(CPUARMState, cp15.csselr_ns) } },
1010     /* Auxiliary ID register: this actually has an IMPDEF value but for now
1011      * just RAZ for all cores:
1012      */
1013     { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
1014       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
1015       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
1016     /* Auxiliary fault status registers: these also are IMPDEF, and we
1017      * choose to RAZ/WI for all cores.
1018      */
1019     { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
1020       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
1021       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
1022     { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
1023       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
1024       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
1025     /* MAIR can just read-as-written because we don't implement caches
1026      * and so don't need to care about memory attributes.
1027      */
1028     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
1029       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
1030       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
1031       .resetvalue = 0 },
1032     /* For non-long-descriptor page tables these are PRRR and NMRR;
1033      * regardless they still act as reads-as-written for QEMU.
1034      */
1035      /* MAIR0/1 are defined separately from their 64-bit counterpart which
1036       * allows them to assign the correct fieldoffset based on the endianness
1037       * handled in the field definitions.
1038       */
1039     { .name = "MAIR0", .state = ARM_CP_STATE_AA32,
1040       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0, .access = PL1_RW,
1041       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
1042                              offsetof(CPUARMState, cp15.mair0_ns) },
1043       .resetfn = arm_cp_reset_ignore },
1044     { .name = "MAIR1", .state = ARM_CP_STATE_AA32,
1045       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1, .access = PL1_RW,
1046       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
1047                              offsetof(CPUARMState, cp15.mair1_ns) },
1048       .resetfn = arm_cp_reset_ignore },
1049     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
1050       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
1051       .type = ARM_CP_NO_RAW, .access = PL1_R, .readfn = isr_read },
1052     /* 32 bit ITLB invalidates */
1053     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
1054       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1055     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
1056       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1057     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
1058       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1059     /* 32 bit DTLB invalidates */
1060     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
1061       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1062     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
1063       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1064     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
1065       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1066     /* 32 bit TLB invalidates */
1067     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
1068       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1069     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
1070       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1071     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
1072       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1073     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
1074       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
1075     REGINFO_SENTINEL
1076 };
1077
1078 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
1079     /* 32 bit TLB invalidates, Inner Shareable */
1080     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
1081       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_is_write },
1082     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
1083       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
1084     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
1085       .type = ARM_CP_NO_RAW, .access = PL1_W,
1086       .writefn = tlbiasid_is_write },
1087     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
1088       .type = ARM_CP_NO_RAW, .access = PL1_W,
1089       .writefn = tlbimvaa_is_write },
1090     REGINFO_SENTINEL
1091 };
1092
1093 static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1094                         uint64_t value)
1095 {
1096     value &= 1;
1097     env->teecr = value;
1098 }
1099
1100 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri)
1101 {
1102     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
1103         return CP_ACCESS_TRAP;
1104     }
1105     return CP_ACCESS_OK;
1106 }
1107
1108 static const ARMCPRegInfo t2ee_cp_reginfo[] = {
1109     { .name = "TEECR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 6, .opc2 = 0,
1110       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, teecr),
1111       .resetvalue = 0,
1112       .writefn = teecr_write },
1113     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
1114       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
1115       .accessfn = teehbr_access, .resetvalue = 0 },
1116     REGINFO_SENTINEL
1117 };
1118
1119 static const ARMCPRegInfo v6k_cp_reginfo[] = {
1120     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
1121       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
1122       .access = PL0_RW,
1123       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
1124     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
1125       .access = PL0_RW,
1126       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
1127                              offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
1128       .resetfn = arm_cp_reset_ignore },
1129     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
1130       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
1131       .access = PL0_R|PL1_W,
1132       .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
1133       .resetvalue = 0},
1134     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
1135       .access = PL0_R|PL1_W,
1136       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
1137                              offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
1138       .resetfn = arm_cp_reset_ignore },
1139     { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
1140       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
1141       .access = PL1_RW,
1142       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
1143     { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
1144       .access = PL1_RW,
1145       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
1146                              offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
1147       .resetvalue = 0 },
1148     REGINFO_SENTINEL
1149 };
1150
1151 #ifndef CONFIG_USER_ONLY
1152
1153 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
1154 {
1155     /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero */
1156     if (arm_current_el(env) == 0 && !extract32(env->cp15.c14_cntkctl, 0, 2)) {
1157         return CP_ACCESS_TRAP;
1158     }
1159     return CP_ACCESS_OK;
1160 }
1161
1162 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx)
1163 {
1164     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
1165     if (arm_current_el(env) == 0 &&
1166         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
1167         return CP_ACCESS_TRAP;
1168     }
1169     return CP_ACCESS_OK;
1170 }
1171
1172 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx)
1173 {
1174     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
1175      * EL0[PV]TEN is zero.
1176      */
1177     if (arm_current_el(env) == 0 &&
1178         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
1179         return CP_ACCESS_TRAP;
1180     }
1181     return CP_ACCESS_OK;
1182 }
1183
1184 static CPAccessResult gt_pct_access(CPUARMState *env,
1185                                          const ARMCPRegInfo *ri)
1186 {
1187     return gt_counter_access(env, GTIMER_PHYS);
1188 }
1189
1190 static CPAccessResult gt_vct_access(CPUARMState *env,
1191                                          const ARMCPRegInfo *ri)
1192 {
1193     return gt_counter_access(env, GTIMER_VIRT);
1194 }
1195
1196 static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
1197 {
1198     return gt_timer_access(env, GTIMER_PHYS);
1199 }
1200
1201 static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
1202 {
1203     return gt_timer_access(env, GTIMER_VIRT);
1204 }
1205
1206 static uint64_t gt_get_countervalue(CPUARMState *env)
1207 {
1208     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / GTIMER_SCALE;
1209 }
1210
1211 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
1212 {
1213     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
1214
1215     if (gt->ctl & 1) {
1216         /* Timer enabled: calculate and set current ISTATUS, irq, and
1217          * reset timer to when ISTATUS next has to change
1218          */
1219         uint64_t count = gt_get_countervalue(&cpu->env);
1220         /* Note that this must be unsigned 64 bit arithmetic: */
1221         int istatus = count >= gt->cval;
1222         uint64_t nexttick;
1223
1224         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
1225         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1226                      (istatus && !(gt->ctl & 2)));
1227         if (istatus) {
1228             /* Next transition is when count rolls back over to zero */
1229             nexttick = UINT64_MAX;
1230         } else {
1231             /* Next transition is when we hit cval */
1232             nexttick = gt->cval;
1233         }
1234         /* Note that the desired next expiry time might be beyond the
1235          * signed-64-bit range of a QEMUTimer -- in this case we just
1236          * set the timer for as far in the future as possible. When the
1237          * timer expires we will reset the timer for any remaining period.
1238          */
1239         if (nexttick > INT64_MAX / GTIMER_SCALE) {
1240             nexttick = INT64_MAX / GTIMER_SCALE;
1241         }
1242         timer_mod(cpu->gt_timer[timeridx], nexttick);
1243     } else {
1244         /* Timer disabled: ISTATUS and timer output always clear */
1245         gt->ctl &= ~4;
1246         qemu_set_irq(cpu->gt_timer_outputs[timeridx], 0);
1247         timer_del(cpu->gt_timer[timeridx]);
1248     }
1249 }
1250
1251 static void gt_cnt_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1252 {
1253     ARMCPU *cpu = arm_env_get_cpu(env);
1254     int timeridx = ri->opc1 & 1;
1255
1256     timer_del(cpu->gt_timer[timeridx]);
1257 }
1258
1259 static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
1260 {
1261     return gt_get_countervalue(env);
1262 }
1263
1264 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1265                           uint64_t value)
1266 {
1267     int timeridx = ri->opc1 & 1;
1268
1269     env->cp15.c14_timer[timeridx].cval = value;
1270     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1271 }
1272
1273 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1274 {
1275     int timeridx = ri->crm & 1;
1276
1277     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
1278                       gt_get_countervalue(env));
1279 }
1280
1281 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1282                           uint64_t value)
1283 {
1284     int timeridx = ri->crm & 1;
1285
1286     env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) +
1287                                          sextract64(value, 0, 32);
1288     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1289 }
1290
1291 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1292                          uint64_t value)
1293 {
1294     ARMCPU *cpu = arm_env_get_cpu(env);
1295     int timeridx = ri->crm & 1;
1296     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
1297
1298     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
1299     if ((oldval ^ value) & 1) {
1300         /* Enable toggled */
1301         gt_recalc_timer(cpu, timeridx);
1302     } else if ((oldval ^ value) & 2) {
1303         /* IMASK toggled: don't need to recalculate,
1304          * just set the interrupt line based on ISTATUS
1305          */
1306         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1307                      (oldval & 4) && !(value & 2));
1308     }
1309 }
1310
1311 void arm_gt_ptimer_cb(void *opaque)
1312 {
1313     ARMCPU *cpu = opaque;
1314
1315     gt_recalc_timer(cpu, GTIMER_PHYS);
1316 }
1317
1318 void arm_gt_vtimer_cb(void *opaque)
1319 {
1320     ARMCPU *cpu = opaque;
1321
1322     gt_recalc_timer(cpu, GTIMER_VIRT);
1323 }
1324
1325 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1326     /* Note that CNTFRQ is purely reads-as-written for the benefit
1327      * of software; writing it doesn't actually change the timer frequency.
1328      * Our reset value matches the fixed frequency we implement the timer at.
1329      */
1330     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
1331       .type = ARM_CP_ALIAS,
1332       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1333       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
1334       .resetfn = arm_cp_reset_ignore,
1335     },
1336     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
1337       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
1338       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1339       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
1340       .resetvalue = (1000 * 1000 * 1000) / GTIMER_SCALE,
1341     },
1342     /* overall control: mostly access permissions */
1343     { .name = "CNTKCTL", .state = ARM_CP_STATE_BOTH,
1344       .opc0 = 3, .opc1 = 0, .crn = 14, .crm = 1, .opc2 = 0,
1345       .access = PL1_RW,
1346       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntkctl),
1347       .resetvalue = 0,
1348     },
1349     /* per-timer control */
1350     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
1351       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1352       .accessfn = gt_ptimer_access,
1353       .fieldoffset = offsetoflow32(CPUARMState,
1354                                    cp15.c14_timer[GTIMER_PHYS].ctl),
1355       .resetfn = arm_cp_reset_ignore,
1356       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1357     },
1358     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
1359       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
1360       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1361       .accessfn = gt_ptimer_access,
1362       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
1363       .resetvalue = 0,
1364       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1365     },
1366     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
1367       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1368       .accessfn = gt_vtimer_access,
1369       .fieldoffset = offsetoflow32(CPUARMState,
1370                                    cp15.c14_timer[GTIMER_VIRT].ctl),
1371       .resetfn = arm_cp_reset_ignore,
1372       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1373     },
1374     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
1375       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
1376       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1377       .accessfn = gt_vtimer_access,
1378       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
1379       .resetvalue = 0,
1380       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1381     },
1382     /* TimerValue views: a 32 bit downcounting view of the underlying state */
1383     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
1384       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1385       .accessfn = gt_ptimer_access,
1386       .readfn = gt_tval_read, .writefn = gt_tval_write,
1387     },
1388     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1389       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
1390       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1391       .accessfn = gt_ptimer_access,
1392       .readfn = gt_tval_read, .writefn = gt_tval_write,
1393     },
1394     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
1395       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1396       .accessfn = gt_vtimer_access,
1397       .readfn = gt_tval_read, .writefn = gt_tval_write,
1398     },
1399     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1400       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
1401       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1402       .accessfn = gt_vtimer_access,
1403       .readfn = gt_tval_read, .writefn = gt_tval_write,
1404     },
1405     /* The counter itself */
1406     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
1407       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1408       .accessfn = gt_pct_access,
1409       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1410     },
1411     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
1412       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
1413       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1414       .accessfn = gt_pct_access,
1415       .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
1416     },
1417     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
1418       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1419       .accessfn = gt_vct_access,
1420       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1421     },
1422     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
1423       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
1424       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1425       .accessfn = gt_vct_access,
1426       .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
1427     },
1428     /* Comparison value, indicating when the timer goes off */
1429     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
1430       .access = PL1_RW | PL0_R,
1431       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1432       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1433       .accessfn = gt_ptimer_access, .resetfn = arm_cp_reset_ignore,
1434       .writefn = gt_cval_write, .raw_writefn = raw_write,
1435     },
1436     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1437       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
1438       .access = PL1_RW | PL0_R,
1439       .type = ARM_CP_IO,
1440       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1441       .resetvalue = 0, .accessfn = gt_ptimer_access,
1442       .writefn = gt_cval_write, .raw_writefn = raw_write,
1443     },
1444     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
1445       .access = PL1_RW | PL0_R,
1446       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1447       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1448       .accessfn = gt_vtimer_access, .resetfn = arm_cp_reset_ignore,
1449       .writefn = gt_cval_write, .raw_writefn = raw_write,
1450     },
1451     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1452       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
1453       .access = PL1_RW | PL0_R,
1454       .type = ARM_CP_IO,
1455       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1456       .resetvalue = 0, .accessfn = gt_vtimer_access,
1457       .writefn = gt_cval_write, .raw_writefn = raw_write,
1458     },
1459     REGINFO_SENTINEL
1460 };
1461
1462 #else
1463 /* In user-mode none of the generic timer registers are accessible,
1464  * and their implementation depends on QEMU_CLOCK_VIRTUAL and qdev gpio outputs,
1465  * so instead just don't register any of them.
1466  */
1467 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1468     REGINFO_SENTINEL
1469 };
1470
1471 #endif
1472
1473 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1474 {
1475     if (arm_feature(env, ARM_FEATURE_LPAE)) {
1476         raw_write(env, ri, value);
1477     } else if (arm_feature(env, ARM_FEATURE_V7)) {
1478         raw_write(env, ri, value & 0xfffff6ff);
1479     } else {
1480         raw_write(env, ri, value & 0xfffff1ff);
1481     }
1482 }
1483
1484 #ifndef CONFIG_USER_ONLY
1485 /* get_phys_addr() isn't present for user-mode-only targets */
1486
1487 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri)
1488 {
1489     if (ri->opc2 & 4) {
1490         /* Other states are only available with TrustZone; in
1491          * a non-TZ implementation these registers don't exist
1492          * at all, which is an Uncategorized trap. This underdecoding
1493          * is safe because the reginfo is NO_RAW.
1494          */
1495         return CP_ACCESS_TRAP_UNCATEGORIZED;
1496     }
1497     return CP_ACCESS_OK;
1498 }
1499
1500 static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
1501                              int access_type, ARMMMUIdx mmu_idx)
1502 {
1503     hwaddr phys_addr;
1504     target_ulong page_size;
1505     int prot;
1506     int ret;
1507     uint64_t par64;
1508     MemTxAttrs attrs = {};
1509
1510     ret = get_phys_addr(env, value, access_type, mmu_idx,
1511                         &phys_addr, &attrs, &prot, &page_size);
1512     if (extended_addresses_enabled(env)) {
1513         /* ret is a DFSR/IFSR value for the long descriptor
1514          * translation table format, but with WnR always clear.
1515          * Convert it to a 64-bit PAR.
1516          */
1517         par64 = (1 << 11); /* LPAE bit always set */
1518         if (ret == 0) {
1519             par64 |= phys_addr & ~0xfffULL;
1520             if (!attrs.secure) {
1521                 par64 |= (1 << 9); /* NS */
1522             }
1523             /* We don't set the ATTR or SH fields in the PAR. */
1524         } else {
1525             par64 |= 1; /* F */
1526             par64 |= (ret & 0x3f) << 1; /* FS */
1527             /* Note that S2WLK and FSTAGE are always zero, because we don't
1528              * implement virtualization and therefore there can't be a stage 2
1529              * fault.
1530              */
1531         }
1532     } else {
1533         /* ret is a DFSR/IFSR value for the short descriptor
1534          * translation table format (with WnR always clear).
1535          * Convert it to a 32-bit PAR.
1536          */
1537         if (ret == 0) {
1538             /* We do not set any attribute bits in the PAR */
1539             if (page_size == (1 << 24)
1540                 && arm_feature(env, ARM_FEATURE_V7)) {
1541                 par64 = (phys_addr & 0xff000000) | (1 << 1);
1542             } else {
1543                 par64 = phys_addr & 0xfffff000;
1544             }
1545             if (!attrs.secure) {
1546                 par64 |= (1 << 9); /* NS */
1547             }
1548         } else {
1549             par64 = ((ret & (1 << 10)) >> 5) | ((ret & (1 << 12)) >> 6) |
1550                     ((ret & 0xf) << 1) | 1;
1551         }
1552     }
1553     return par64;
1554 }
1555
1556 static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1557 {
1558     int access_type = ri->opc2 & 1;
1559     uint64_t par64;
1560     ARMMMUIdx mmu_idx;
1561     int el = arm_current_el(env);
1562     bool secure = arm_is_secure_below_el3(env);
1563
1564     switch (ri->opc2 & 6) {
1565     case 0:
1566         /* stage 1 current state PL1: ATS1CPR, ATS1CPW */
1567         switch (el) {
1568         case 3:
1569             mmu_idx = ARMMMUIdx_S1E3;
1570             break;
1571         case 2:
1572             mmu_idx = ARMMMUIdx_S1NSE1;
1573             break;
1574         case 1:
1575             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
1576             break;
1577         default:
1578             g_assert_not_reached();
1579         }
1580         break;
1581     case 2:
1582         /* stage 1 current state PL0: ATS1CUR, ATS1CUW */
1583         switch (el) {
1584         case 3:
1585             mmu_idx = ARMMMUIdx_S1SE0;
1586             break;
1587         case 2:
1588             mmu_idx = ARMMMUIdx_S1NSE0;
1589             break;
1590         case 1:
1591             mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
1592             break;
1593         default:
1594             g_assert_not_reached();
1595         }
1596         break;
1597     case 4:
1598         /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */
1599         mmu_idx = ARMMMUIdx_S12NSE1;
1600         break;
1601     case 6:
1602         /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */
1603         mmu_idx = ARMMMUIdx_S12NSE0;
1604         break;
1605     default:
1606         g_assert_not_reached();
1607     }
1608
1609     par64 = do_ats_write(env, value, access_type, mmu_idx);
1610
1611     A32_BANKED_CURRENT_REG_SET(env, par, par64);
1612 }
1613
1614 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
1615                         uint64_t value)
1616 {
1617     int access_type = ri->opc2 & 1;
1618     ARMMMUIdx mmu_idx;
1619     int secure = arm_is_secure_below_el3(env);
1620
1621     switch (ri->opc2 & 6) {
1622     case 0:
1623         switch (ri->opc1) {
1624         case 0: /* AT S1E1R, AT S1E1W */
1625             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
1626             break;
1627         case 4: /* AT S1E2R, AT S1E2W */
1628             mmu_idx = ARMMMUIdx_S1E2;
1629             break;
1630         case 6: /* AT S1E3R, AT S1E3W */
1631             mmu_idx = ARMMMUIdx_S1E3;
1632             break;
1633         default:
1634             g_assert_not_reached();
1635         }
1636         break;
1637     case 2: /* AT S1E0R, AT S1E0W */
1638         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
1639         break;
1640     case 4: /* AT S12E1R, AT S12E1W */
1641         mmu_idx = ARMMMUIdx_S12NSE1;
1642         break;
1643     case 6: /* AT S12E0R, AT S12E0W */
1644         mmu_idx = ARMMMUIdx_S12NSE0;
1645         break;
1646     default:
1647         g_assert_not_reached();
1648     }
1649
1650     env->cp15.par_el[1] = do_ats_write(env, value, access_type, mmu_idx);
1651 }
1652 #endif
1653
1654 static const ARMCPRegInfo vapa_cp_reginfo[] = {
1655     { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
1656       .access = PL1_RW, .resetvalue = 0,
1657       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
1658                              offsetoflow32(CPUARMState, cp15.par_ns) },
1659       .writefn = par_write },
1660 #ifndef CONFIG_USER_ONLY
1661     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
1662       .access = PL1_W, .accessfn = ats_access,
1663       .writefn = ats_write, .type = ARM_CP_NO_RAW },
1664 #endif
1665     REGINFO_SENTINEL
1666 };
1667
1668 /* Return basic MPU access permission bits.  */
1669 static uint32_t simple_mpu_ap_bits(uint32_t val)
1670 {
1671     uint32_t ret;
1672     uint32_t mask;
1673     int i;
1674     ret = 0;
1675     mask = 3;
1676     for (i = 0; i < 16; i += 2) {
1677         ret |= (val >> i) & mask;
1678         mask <<= 2;
1679     }
1680     return ret;
1681 }
1682
1683 /* Pad basic MPU access permission bits to extended format.  */
1684 static uint32_t extended_mpu_ap_bits(uint32_t val)
1685 {
1686     uint32_t ret;
1687     uint32_t mask;
1688     int i;
1689     ret = 0;
1690     mask = 3;
1691     for (i = 0; i < 16; i += 2) {
1692         ret |= (val & mask) << i;
1693         mask <<= 2;
1694     }
1695     return ret;
1696 }
1697
1698 static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
1699                                  uint64_t value)
1700 {
1701     env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
1702 }
1703
1704 static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
1705 {
1706     return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
1707 }
1708
1709 static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
1710                                  uint64_t value)
1711 {
1712     env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
1713 }
1714
1715 static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
1716 {
1717     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
1718 }
1719
1720 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
1721     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
1722       .access = PL1_RW, .type = ARM_CP_ALIAS,
1723       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
1724       .resetvalue = 0,
1725       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
1726     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
1727       .access = PL1_RW, .type = ARM_CP_ALIAS,
1728       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
1729       .resetvalue = 0,
1730       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
1731     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
1732       .access = PL1_RW,
1733       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
1734       .resetvalue = 0, },
1735     { .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
1736       .access = PL1_RW,
1737       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
1738       .resetvalue = 0, },
1739     { .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
1740       .access = PL1_RW,
1741       .fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
1742     { .name = "ICACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
1743       .access = PL1_RW,
1744       .fieldoffset = offsetof(CPUARMState, cp15.c2_insn), .resetvalue = 0, },
1745     /* Protection region base and size registers */
1746     { .name = "946_PRBS0", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0,
1747       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1748       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[0]) },
1749     { .name = "946_PRBS1", .cp = 15, .crn = 6, .crm = 1, .opc1 = 0,
1750       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1751       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[1]) },
1752     { .name = "946_PRBS2", .cp = 15, .crn = 6, .crm = 2, .opc1 = 0,
1753       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1754       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[2]) },
1755     { .name = "946_PRBS3", .cp = 15, .crn = 6, .crm = 3, .opc1 = 0,
1756       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1757       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[3]) },
1758     { .name = "946_PRBS4", .cp = 15, .crn = 6, .crm = 4, .opc1 = 0,
1759       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1760       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[4]) },
1761     { .name = "946_PRBS5", .cp = 15, .crn = 6, .crm = 5, .opc1 = 0,
1762       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1763       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[5]) },
1764     { .name = "946_PRBS6", .cp = 15, .crn = 6, .crm = 6, .opc1 = 0,
1765       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1766       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[6]) },
1767     { .name = "946_PRBS7", .cp = 15, .crn = 6, .crm = 7, .opc1 = 0,
1768       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1769       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[7]) },
1770     REGINFO_SENTINEL
1771 };
1772
1773 static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
1774                                  uint64_t value)
1775 {
1776     TCR *tcr = raw_ptr(env, ri);
1777     int maskshift = extract32(value, 0, 3);
1778
1779     if (!arm_feature(env, ARM_FEATURE_V8)) {
1780         if (arm_feature(env, ARM_FEATURE_LPAE) && (value & TTBCR_EAE)) {
1781             /* Pre ARMv8 bits [21:19], [15:14] and [6:3] are UNK/SBZP when
1782              * using Long-desciptor translation table format */
1783             value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
1784         } else if (arm_feature(env, ARM_FEATURE_EL3)) {
1785             /* In an implementation that includes the Security Extensions
1786              * TTBCR has additional fields PD0 [4] and PD1 [5] for
1787              * Short-descriptor translation table format.
1788              */
1789             value &= TTBCR_PD1 | TTBCR_PD0 | TTBCR_N;
1790         } else {
1791             value &= TTBCR_N;
1792         }
1793     }
1794
1795     /* Update the masks corresponding to the the TCR bank being written
1796      * Note that we always calculate mask and base_mask, but
1797      * they are only used for short-descriptor tables (ie if EAE is 0);
1798      * for long-descriptor tables the TCR fields are used differently
1799      * and the mask and base_mask values are meaningless.
1800      */
1801     tcr->raw_tcr = value;
1802     tcr->mask = ~(((uint32_t)0xffffffffu) >> maskshift);
1803     tcr->base_mask = ~((uint32_t)0x3fffu >> maskshift);
1804 }
1805
1806 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1807                              uint64_t value)
1808 {
1809     ARMCPU *cpu = arm_env_get_cpu(env);
1810
1811     if (arm_feature(env, ARM_FEATURE_LPAE)) {
1812         /* With LPAE the TTBCR could result in a change of ASID
1813          * via the TTBCR.A1 bit, so do a TLB flush.
1814          */
1815         tlb_flush(CPU(cpu), 1);
1816     }
1817     vmsa_ttbcr_raw_write(env, ri, value);
1818 }
1819
1820 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1821 {
1822     TCR *tcr = raw_ptr(env, ri);
1823
1824     /* Reset both the TCR as well as the masks corresponding to the bank of
1825      * the TCR being reset.
1826      */
1827     tcr->raw_tcr = 0;
1828     tcr->mask = 0;
1829     tcr->base_mask = 0xffffc000u;
1830 }
1831
1832 static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri,
1833                                uint64_t value)
1834 {
1835     ARMCPU *cpu = arm_env_get_cpu(env);
1836     TCR *tcr = raw_ptr(env, ri);
1837
1838     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
1839     tlb_flush(CPU(cpu), 1);
1840     tcr->raw_tcr = value;
1841 }
1842
1843 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1844                             uint64_t value)
1845 {
1846     /* 64 bit accesses to the TTBRs can change the ASID and so we
1847      * must flush the TLB.
1848      */
1849     if (cpreg_field_is_64bit(ri)) {
1850         ARMCPU *cpu = arm_env_get_cpu(env);
1851
1852         tlb_flush(CPU(cpu), 1);
1853     }
1854     raw_write(env, ri, value);
1855 }
1856
1857 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
1858     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
1859       .access = PL1_RW, .type = ARM_CP_ALIAS,
1860       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
1861                              offsetoflow32(CPUARMState, cp15.dfsr_ns) },
1862       .resetfn = arm_cp_reset_ignore, },
1863     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
1864       .access = PL1_RW, .resetvalue = 0,
1865       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
1866                              offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
1867     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
1868       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
1869       .access = PL1_RW,
1870       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
1871     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
1872       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
1873       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
1874       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
1875                              offsetof(CPUARMState, cp15.ttbr0_ns) } },
1876     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
1877       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
1878       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
1879       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
1880                              offsetof(CPUARMState, cp15.ttbr1_ns) } },
1881     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
1882       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
1883       .access = PL1_RW, .writefn = vmsa_tcr_el1_write,
1884       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
1885       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
1886     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
1887       .access = PL1_RW, .type = ARM_CP_ALIAS, .writefn = vmsa_ttbcr_write,
1888       .resetfn = arm_cp_reset_ignore, .raw_writefn = vmsa_ttbcr_raw_write,
1889       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
1890                              offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
1891     { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
1892       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
1893       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
1894       .resetvalue = 0, },
1895     { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
1896       .access = PL1_RW, .resetvalue = 0,
1897       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
1898                              offsetof(CPUARMState, cp15.dfar_ns) } },
1899     REGINFO_SENTINEL
1900 };
1901
1902 static void omap_ticonfig_write(CPUARMState *env, const ARMCPRegInfo *ri,
1903                                 uint64_t value)
1904 {
1905     env->cp15.c15_ticonfig = value & 0xe7;
1906     /* The OS_TYPE bit in this register changes the reported CPUID! */
1907     env->cp15.c0_cpuid = (value & (1 << 5)) ?
1908         ARM_CPUID_TI915T : ARM_CPUID_TI925T;
1909 }
1910
1911 static void omap_threadid_write(CPUARMState *env, const ARMCPRegInfo *ri,
1912                                 uint64_t value)
1913 {
1914     env->cp15.c15_threadid = value & 0xffff;
1915 }
1916
1917 static void omap_wfi_write(CPUARMState *env, const ARMCPRegInfo *ri,
1918                            uint64_t value)
1919 {
1920     /* Wait-for-interrupt (deprecated) */
1921     cpu_interrupt(CPU(arm_env_get_cpu(env)), CPU_INTERRUPT_HALT);
1922 }
1923
1924 static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
1925                                   uint64_t value)
1926 {
1927     /* On OMAP there are registers indicating the max/min index of dcache lines
1928      * containing a dirty line; cache flush operations have to reset these.
1929      */
1930     env->cp15.c15_i_max = 0x000;
1931     env->cp15.c15_i_min = 0xff0;
1932 }
1933
1934 static const ARMCPRegInfo omap_cp_reginfo[] = {
1935     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
1936       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
1937       .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
1938       .resetvalue = 0, },
1939     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
1940       .access = PL1_RW, .type = ARM_CP_NOP },
1941     { .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
1942       .access = PL1_RW,
1943       .fieldoffset = offsetof(CPUARMState, cp15.c15_ticonfig), .resetvalue = 0,
1944       .writefn = omap_ticonfig_write },
1945     { .name = "IMAX", .cp = 15, .crn = 15, .crm = 2, .opc1 = 0, .opc2 = 0,
1946       .access = PL1_RW,
1947       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_max), .resetvalue = 0, },
1948     { .name = "IMIN", .cp = 15, .crn = 15, .crm = 3, .opc1 = 0, .opc2 = 0,
1949       .access = PL1_RW, .resetvalue = 0xff0,
1950       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_min) },
1951     { .name = "THREADID", .cp = 15, .crn = 15, .crm = 4, .opc1 = 0, .opc2 = 0,
1952       .access = PL1_RW,
1953       .fieldoffset = offsetof(CPUARMState, cp15.c15_threadid), .resetvalue = 0,
1954       .writefn = omap_threadid_write },
1955     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
1956       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
1957       .type = ARM_CP_NO_RAW,
1958       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
1959     /* TODO: Peripheral port remap register:
1960      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
1961      * base address at $rn & ~0xfff and map size of 0x200 << ($rn & 0xfff),
1962      * when MMU is off.
1963      */
1964     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
1965       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
1966       .type = ARM_CP_OVERRIDE | ARM_CP_NO_RAW,
1967       .writefn = omap_cachemaint_write },
1968     { .name = "C9", .cp = 15, .crn = 9,
1969       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
1970       .type = ARM_CP_CONST | ARM_CP_OVERRIDE, .resetvalue = 0 },
1971     REGINFO_SENTINEL
1972 };
1973
1974 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
1975                               uint64_t value)
1976 {
1977     env->cp15.c15_cpar = value & 0x3fff;
1978 }
1979
1980 static const ARMCPRegInfo xscale_cp_reginfo[] = {
1981     { .name = "XSCALE_CPAR",
1982       .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
1983       .fieldoffset = offsetof(CPUARMState, cp15.c15_cpar), .resetvalue = 0,
1984       .writefn = xscale_cpar_write, },
1985     { .name = "XSCALE_AUXCR",
1986       .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW,
1987       .fieldoffset = offsetof(CPUARMState, cp15.c1_xscaleauxcr),
1988       .resetvalue = 0, },
1989     /* XScale specific cache-lockdown: since we have no cache we NOP these
1990      * and hope the guest does not really rely on cache behaviour.
1991      */
1992     { .name = "XSCALE_LOCK_ICACHE_LINE",
1993       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
1994       .access = PL1_W, .type = ARM_CP_NOP },
1995     { .name = "XSCALE_UNLOCK_ICACHE",
1996       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
1997       .access = PL1_W, .type = ARM_CP_NOP },
1998     { .name = "XSCALE_DCACHE_LOCK",
1999       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 0,
2000       .access = PL1_RW, .type = ARM_CP_NOP },
2001     { .name = "XSCALE_UNLOCK_DCACHE",
2002       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 1,
2003       .access = PL1_W, .type = ARM_CP_NOP },
2004     REGINFO_SENTINEL
2005 };
2006
2007 static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
2008     /* RAZ/WI the whole crn=15 space, when we don't have a more specific
2009      * implementation of this implementation-defined space.
2010      * Ideally this should eventually disappear in favour of actually
2011      * implementing the correct behaviour for all cores.
2012      */
2013     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
2014       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
2015       .access = PL1_RW,
2016       .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
2017       .resetvalue = 0 },
2018     REGINFO_SENTINEL
2019 };
2020
2021 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
2022     /* Cache status: RAZ because we have no cache so it's always clean */
2023     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
2024       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2025       .resetvalue = 0 },
2026     REGINFO_SENTINEL
2027 };
2028
2029 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
2030     /* We never have a a block transfer operation in progress */
2031     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
2032       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2033       .resetvalue = 0 },
2034     /* The cache ops themselves: these all NOP for QEMU */
2035     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
2036       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2037     { .name = "IDCR", .cp = 15, .crm = 6, .opc1 = 0,
2038       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2039     { .name = "CDCR", .cp = 15, .crm = 12, .opc1 = 0,
2040       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2041     { .name = "PIR", .cp = 15, .crm = 12, .opc1 = 1,
2042       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2043     { .name = "PDR", .cp = 15, .crm = 12, .opc1 = 2,
2044       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2045     { .name = "CIDCR", .cp = 15, .crm = 14, .opc1 = 0,
2046       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2047     REGINFO_SENTINEL
2048 };
2049
2050 static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
2051     /* The cache test-and-clean instructions always return (1 << 30)
2052      * to indicate that there are no dirty cache lines.
2053      */
2054     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
2055       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2056       .resetvalue = (1 << 30) },
2057     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
2058       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2059       .resetvalue = (1 << 30) },
2060     REGINFO_SENTINEL
2061 };
2062
2063 static const ARMCPRegInfo strongarm_cp_reginfo[] = {
2064     /* Ignore ReadBuffer accesses */
2065     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
2066       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
2067       .access = PL1_RW, .resetvalue = 0,
2068       .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_RAW },
2069     REGINFO_SENTINEL
2070 };
2071
2072 static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2073 {
2074     CPUState *cs = CPU(arm_env_get_cpu(env));
2075     uint32_t mpidr = cs->cpu_index;
2076     /* We don't support setting cluster ID ([8..11]) (known as Aff1
2077      * in later ARM ARM versions), or any of the higher affinity level fields,
2078      * so these bits always RAZ.
2079      */
2080     if (arm_feature(env, ARM_FEATURE_V7MP)) {
2081         mpidr |= (1U << 31);
2082         /* Cores which are uniprocessor (non-coherent)
2083          * but still implement the MP extensions set
2084          * bit 30. (For instance, A9UP.) However we do
2085          * not currently model any of those cores.
2086          */
2087     }
2088     return mpidr;
2089 }
2090
2091 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
2092     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
2093       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
2094       .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_RAW },
2095     REGINFO_SENTINEL
2096 };
2097
2098 static const ARMCPRegInfo lpae_cp_reginfo[] = {
2099     /* NOP AMAIR0/1 */
2100     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
2101       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
2102       .access = PL1_RW, .type = ARM_CP_CONST,
2103       .resetvalue = 0 },
2104     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
2105     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
2106       .access = PL1_RW, .type = ARM_CP_CONST,
2107       .resetvalue = 0 },
2108     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
2109       .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
2110       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
2111                              offsetof(CPUARMState, cp15.par_ns)} },
2112     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
2113       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2114       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
2115                              offsetof(CPUARMState, cp15.ttbr0_ns) },
2116       .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
2117     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
2118       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2119       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
2120                              offsetof(CPUARMState, cp15.ttbr1_ns) },
2121       .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
2122     REGINFO_SENTINEL
2123 };
2124
2125 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2126 {
2127     return vfp_get_fpcr(env);
2128 }
2129
2130 static void aa64_fpcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2131                             uint64_t value)
2132 {
2133     vfp_set_fpcr(env, value);
2134 }
2135
2136 static uint64_t aa64_fpsr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2137 {
2138     return vfp_get_fpsr(env);
2139 }
2140
2141 static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2142                             uint64_t value)
2143 {
2144     vfp_set_fpsr(env, value);
2145 }
2146
2147 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri)
2148 {
2149     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
2150         return CP_ACCESS_TRAP;
2151     }
2152     return CP_ACCESS_OK;
2153 }
2154
2155 static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
2156                             uint64_t value)
2157 {
2158     env->daif = value & PSTATE_DAIF;
2159 }
2160
2161 static CPAccessResult aa64_cacheop_access(CPUARMState *env,
2162                                           const ARMCPRegInfo *ri)
2163 {
2164     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
2165      * SCTLR_EL1.UCI is set.
2166      */
2167     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) {
2168         return CP_ACCESS_TRAP;
2169     }
2170     return CP_ACCESS_OK;
2171 }
2172
2173 /* See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
2174  * Page D4-1736 (DDI0487A.b)
2175  */
2176
2177 static void tlbi_aa64_va_write(CPUARMState *env, const ARMCPRegInfo *ri,
2178                                uint64_t value)
2179 {
2180     /* Invalidate by VA (AArch64 version) */
2181     ARMCPU *cpu = arm_env_get_cpu(env);
2182     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2183
2184     tlb_flush_page(CPU(cpu), pageaddr);
2185 }
2186
2187 static void tlbi_aa64_vaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
2188                                 uint64_t value)
2189 {
2190     /* Invalidate by VA, all ASIDs (AArch64 version) */
2191     ARMCPU *cpu = arm_env_get_cpu(env);
2192     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2193
2194     tlb_flush_page(CPU(cpu), pageaddr);
2195 }
2196
2197 static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
2198                                  uint64_t value)
2199 {
2200     /* Invalidate by ASID (AArch64 version) */
2201     ARMCPU *cpu = arm_env_get_cpu(env);
2202     int asid = extract64(value, 48, 16);
2203     tlb_flush(CPU(cpu), asid == 0);
2204 }
2205
2206 static void tlbi_aa64_va_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2207                                   uint64_t value)
2208 {
2209     CPUState *other_cs;
2210     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2211
2212     CPU_FOREACH(other_cs) {
2213         tlb_flush_page(other_cs, pageaddr);
2214     }
2215 }
2216
2217 static void tlbi_aa64_vaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2218                                   uint64_t value)
2219 {
2220     CPUState *other_cs;
2221     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2222
2223     CPU_FOREACH(other_cs) {
2224         tlb_flush_page(other_cs, pageaddr);
2225     }
2226 }
2227
2228 static void tlbi_aa64_asid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2229                                   uint64_t value)
2230 {
2231     CPUState *other_cs;
2232     int asid = extract64(value, 48, 16);
2233
2234     CPU_FOREACH(other_cs) {
2235         tlb_flush(other_cs, asid == 0);
2236     }
2237 }
2238
2239 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri)
2240 {
2241     /* We don't implement EL2, so the only control on DC ZVA is the
2242      * bit in the SCTLR which can prohibit access for EL0.
2243      */
2244     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
2245         return CP_ACCESS_TRAP;
2246     }
2247     return CP_ACCESS_OK;
2248 }
2249
2250 static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
2251 {
2252     ARMCPU *cpu = arm_env_get_cpu(env);
2253     int dzp_bit = 1 << 4;
2254
2255     /* DZP indicates whether DC ZVA access is allowed */
2256     if (aa64_zva_access(env, NULL) == CP_ACCESS_OK) {
2257         dzp_bit = 0;
2258     }
2259     return cpu->dcz_blocksize | dzp_bit;
2260 }
2261
2262 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
2263 {
2264     if (!(env->pstate & PSTATE_SP)) {
2265         /* Access to SP_EL0 is undefined if it's being used as
2266          * the stack pointer.
2267          */
2268         return CP_ACCESS_TRAP_UNCATEGORIZED;
2269     }
2270     return CP_ACCESS_OK;
2271 }
2272
2273 static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
2274 {
2275     return env->pstate & PSTATE_SP;
2276 }
2277
2278 static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
2279 {
2280     update_spsel(env, val);
2281 }
2282
2283 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2284                         uint64_t value)
2285 {
2286     ARMCPU *cpu = arm_env_get_cpu(env);
2287
2288     if (raw_read(env, ri) == value) {
2289         /* Skip the TLB flush if nothing actually changed; Linux likes
2290          * to do a lot of pointless SCTLR writes.
2291          */
2292         return;
2293     }
2294
2295     raw_write(env, ri, value);
2296     /* ??? Lots of these bits are not implemented.  */
2297     /* This may enable/disable the MMU, so do a TLB flush.  */
2298     tlb_flush(CPU(cpu), 1);
2299 }
2300
2301 static const ARMCPRegInfo v8_cp_reginfo[] = {
2302     /* Minimal set of EL0-visible registers. This will need to be expanded
2303      * significantly for system emulation of AArch64 CPUs.
2304      */
2305     { .name = "NZCV", .state = ARM_CP_STATE_AA64,
2306       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
2307       .access = PL0_RW, .type = ARM_CP_NZCV },
2308     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
2309       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
2310       .type = ARM_CP_NO_RAW,
2311       .access = PL0_RW, .accessfn = aa64_daif_access,
2312       .fieldoffset = offsetof(CPUARMState, daif),
2313       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
2314     { .name = "FPCR", .state = ARM_CP_STATE_AA64,
2315       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
2316       .access = PL0_RW, .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
2317     { .name = "FPSR", .state = ARM_CP_STATE_AA64,
2318       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
2319       .access = PL0_RW, .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
2320     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
2321       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
2322       .access = PL0_R, .type = ARM_CP_NO_RAW,
2323       .readfn = aa64_dczid_read },
2324     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
2325       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
2326       .access = PL0_W, .type = ARM_CP_DC_ZVA,
2327 #ifndef CONFIG_USER_ONLY
2328       /* Avoid overhead of an access check that always passes in user-mode */
2329       .accessfn = aa64_zva_access,
2330 #endif
2331     },
2332     { .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
2333       .opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
2334       .access = PL1_R, .type = ARM_CP_CURRENTEL },
2335     /* Cache ops: all NOPs since we don't emulate caches */
2336     { .name = "IC_IALLUIS", .state = ARM_CP_STATE_AA64,
2337       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
2338       .access = PL1_W, .type = ARM_CP_NOP },
2339     { .name = "IC_IALLU", .state = ARM_CP_STATE_AA64,
2340       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
2341       .access = PL1_W, .type = ARM_CP_NOP },
2342     { .name = "IC_IVAU", .state = ARM_CP_STATE_AA64,
2343       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 5, .opc2 = 1,
2344       .access = PL0_W, .type = ARM_CP_NOP,
2345       .accessfn = aa64_cacheop_access },
2346     { .name = "DC_IVAC", .state = ARM_CP_STATE_AA64,
2347       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
2348       .access = PL1_W, .type = ARM_CP_NOP },
2349     { .name = "DC_ISW", .state = ARM_CP_STATE_AA64,
2350       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
2351       .access = PL1_W, .type = ARM_CP_NOP },
2352     { .name = "DC_CVAC", .state = ARM_CP_STATE_AA64,
2353       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 1,
2354       .access = PL0_W, .type = ARM_CP_NOP,
2355       .accessfn = aa64_cacheop_access },
2356     { .name = "DC_CSW", .state = ARM_CP_STATE_AA64,
2357       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
2358       .access = PL1_W, .type = ARM_CP_NOP },
2359     { .name = "DC_CVAU", .state = ARM_CP_STATE_AA64,
2360       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 11, .opc2 = 1,
2361       .access = PL0_W, .type = ARM_CP_NOP,
2362       .accessfn = aa64_cacheop_access },
2363     { .name = "DC_CIVAC", .state = ARM_CP_STATE_AA64,
2364       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 1,
2365       .access = PL0_W, .type = ARM_CP_NOP,
2366       .accessfn = aa64_cacheop_access },
2367     { .name = "DC_CISW", .state = ARM_CP_STATE_AA64,
2368       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
2369       .access = PL1_W, .type = ARM_CP_NOP },
2370     /* TLBI operations */
2371     { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
2372       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
2373       .access = PL2_W, .type = ARM_CP_NO_RAW,
2374       .writefn = tlbiall_write },
2375     { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
2376       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
2377       .access = PL2_W, .type = ARM_CP_NO_RAW,
2378       .writefn = tlbiall_write },
2379     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
2380       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
2381       .access = PL1_W, .type = ARM_CP_NO_RAW,
2382       .writefn = tlbiall_is_write },
2383     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
2384       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
2385       .access = PL1_W, .type = ARM_CP_NO_RAW,
2386       .writefn = tlbi_aa64_va_is_write },
2387     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
2388       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
2389       .access = PL1_W, .type = ARM_CP_NO_RAW,
2390       .writefn = tlbi_aa64_asid_is_write },
2391     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
2392       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
2393       .access = PL1_W, .type = ARM_CP_NO_RAW,
2394       .writefn = tlbi_aa64_vaa_is_write },
2395     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
2396       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
2397       .access = PL1_W, .type = ARM_CP_NO_RAW,
2398       .writefn = tlbi_aa64_va_is_write },
2399     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
2400       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
2401       .access = PL1_W, .type = ARM_CP_NO_RAW,
2402       .writefn = tlbi_aa64_vaa_is_write },
2403     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
2404       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
2405       .access = PL1_W, .type = ARM_CP_NO_RAW,
2406       .writefn = tlbiall_write },
2407     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
2408       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
2409       .access = PL1_W, .type = ARM_CP_NO_RAW,
2410       .writefn = tlbi_aa64_va_write },
2411     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
2412       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
2413       .access = PL1_W, .type = ARM_CP_NO_RAW,
2414       .writefn = tlbi_aa64_asid_write },
2415     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
2416       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
2417       .access = PL1_W, .type = ARM_CP_NO_RAW,
2418       .writefn = tlbi_aa64_vaa_write },
2419     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
2420       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
2421       .access = PL1_W, .type = ARM_CP_NO_RAW,
2422       .writefn = tlbi_aa64_va_write },
2423     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
2424       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
2425       .access = PL1_W, .type = ARM_CP_NO_RAW,
2426       .writefn = tlbi_aa64_vaa_write },
2427 #ifndef CONFIG_USER_ONLY
2428     /* 64 bit address translation operations */
2429     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
2430       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
2431       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2432     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
2433       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
2434       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2435     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
2436       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
2437       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2438     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
2439       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
2440       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2441 #endif
2442     /* TLB invalidate last level of translation table walk */
2443     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
2444       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
2445     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
2446       .type = ARM_CP_NO_RAW, .access = PL1_W,
2447       .writefn = tlbimvaa_is_write },
2448     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
2449       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
2450     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
2451       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
2452     /* 32 bit cache operations */
2453     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
2454       .type = ARM_CP_NOP, .access = PL1_W },
2455     { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
2456       .type = ARM_CP_NOP, .access = PL1_W },
2457     { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
2458       .type = ARM_CP_NOP, .access = PL1_W },
2459     { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
2460       .type = ARM_CP_NOP, .access = PL1_W },
2461     { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
2462       .type = ARM_CP_NOP, .access = PL1_W },
2463     { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
2464       .type = ARM_CP_NOP, .access = PL1_W },
2465     { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
2466       .type = ARM_CP_NOP, .access = PL1_W },
2467     { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
2468       .type = ARM_CP_NOP, .access = PL1_W },
2469     { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
2470       .type = ARM_CP_NOP, .access = PL1_W },
2471     { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
2472       .type = ARM_CP_NOP, .access = PL1_W },
2473     { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
2474       .type = ARM_CP_NOP, .access = PL1_W },
2475     { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
2476       .type = ARM_CP_NOP, .access = PL1_W },
2477     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
2478       .type = ARM_CP_NOP, .access = PL1_W },
2479     /* MMU Domain access control / MPU write buffer control */
2480     { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
2481       .access = PL1_RW, .resetvalue = 0,
2482       .writefn = dacr_write, .raw_writefn = raw_write,
2483       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
2484                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
2485     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
2486       .type = ARM_CP_ALIAS,
2487       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
2488       .access = PL1_RW,
2489       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
2490     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
2491       .type = ARM_CP_ALIAS,
2492       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
2493       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[1]) },
2494     /* We rely on the access checks not allowing the guest to write to the
2495      * state field when SPSel indicates that it's being used as the stack
2496      * pointer.
2497      */
2498     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
2499       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
2500       .access = PL1_RW, .accessfn = sp_el0_access,
2501       .type = ARM_CP_ALIAS,
2502       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
2503     { .name = "SP_EL1", .state = ARM_CP_STATE_AA64,
2504       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 1, .opc2 = 0,
2505       .access = PL2_RW, .type = ARM_CP_ALIAS,
2506       .fieldoffset = offsetof(CPUARMState, sp_el[1]) },
2507     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
2508       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
2509       .type = ARM_CP_NO_RAW,
2510       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
2511     REGINFO_SENTINEL
2512 };
2513
2514 /* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
2515 static const ARMCPRegInfo v8_el3_no_el2_cp_reginfo[] = {
2516     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
2517       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
2518       .access = PL2_RW,
2519       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
2520     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
2521       .type = ARM_CP_NO_RAW,
2522       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
2523       .access = PL2_RW,
2524       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
2525     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
2526       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
2527       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
2528     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
2529       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
2530       .access = PL2_RW, .type = ARM_CP_CONST,
2531       .resetvalue = 0 },
2532     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
2533       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
2534       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
2535     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
2536       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
2537       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
2538     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
2539       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
2540       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
2541     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
2542       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
2543       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
2544     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
2545       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
2546       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
2547     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
2548       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
2549       .resetvalue = 0 },
2550     REGINFO_SENTINEL
2551 };
2552
2553 static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
2554 {
2555     ARMCPU *cpu = arm_env_get_cpu(env);
2556     uint64_t valid_mask = HCR_MASK;
2557
2558     if (arm_feature(env, ARM_FEATURE_EL3)) {
2559         valid_mask &= ~HCR_HCD;
2560     } else {
2561         valid_mask &= ~HCR_TSC;
2562     }
2563
2564     /* Clear RES0 bits.  */
2565     value &= valid_mask;
2566
2567     /* These bits change the MMU setup:
2568      * HCR_VM enables stage 2 translation
2569      * HCR_PTW forbids certain page-table setups
2570      * HCR_DC Disables stage1 and enables stage2 translation
2571      */
2572     if ((raw_read(env, ri) ^ value) & (HCR_VM | HCR_PTW | HCR_DC)) {
2573         tlb_flush(CPU(cpu), 1);
2574     }
2575     raw_write(env, ri, value);
2576 }
2577
2578 static const ARMCPRegInfo v8_el2_cp_reginfo[] = {
2579     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
2580       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
2581       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
2582       .writefn = hcr_write },
2583     { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
2584       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
2585       .access = PL2_RW, .resetvalue = 0,
2586       .writefn = dacr_write, .raw_writefn = raw_write,
2587       .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
2588     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
2589       .type = ARM_CP_ALIAS,
2590       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
2591       .access = PL2_RW,
2592       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
2593     { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
2594       .type = ARM_CP_ALIAS,
2595       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
2596       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
2597     { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
2598       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
2599       .access = PL2_RW, .resetvalue = 0,
2600       .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
2601     { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
2602       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
2603       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
2604     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
2605       .type = ARM_CP_ALIAS,
2606       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
2607       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[6]) },
2608     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
2609       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
2610       .access = PL2_RW, .writefn = vbar_write,
2611       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
2612       .resetvalue = 0 },
2613     { .name = "SP_EL2", .state = ARM_CP_STATE_AA64,
2614       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 1, .opc2 = 0,
2615       .access = PL3_RW, .type = ARM_CP_ALIAS,
2616       .fieldoffset = offsetof(CPUARMState, sp_el[2]) },
2617     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
2618       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
2619       .access = PL2_RW, .accessfn = cptr_access, .resetvalue = 0,
2620       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[2]) },
2621     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
2622       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
2623       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[2]),
2624       .resetvalue = 0 },
2625     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
2626       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
2627       .access = PL2_RW, .type = ARM_CP_ALIAS,
2628       .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
2629     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
2630       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
2631       .access = PL2_RW, .writefn = vmsa_tcr_el1_write,
2632       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
2633       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
2634     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
2635       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
2636       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
2637       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[2]) },
2638     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
2639       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
2640       .access = PL2_RW, .resetvalue = 0,
2641       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) },
2642     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
2643       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
2644       .access = PL2_RW, .resetvalue = 0,
2645       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
2646     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
2647       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2648       .resetvalue = 0,
2649       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
2650     { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
2651       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
2652       .type = ARM_CP_NO_RAW, .access = PL2_W,
2653       .writefn = tlbiall_write },
2654     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
2655       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
2656       .type = ARM_CP_NO_RAW, .access = PL2_W,
2657       .writefn = tlbi_aa64_vaa_write },
2658     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
2659       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
2660       .type = ARM_CP_NO_RAW, .access = PL2_W,
2661       .writefn = tlbi_aa64_vaa_write },
2662     REGINFO_SENTINEL
2663 };
2664
2665 static const ARMCPRegInfo el3_cp_reginfo[] = {
2666     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
2667       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
2668       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
2669       .resetvalue = 0, .writefn = scr_write },
2670     { .name = "SCR",  .type = ARM_CP_ALIAS,
2671       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
2672       .access = PL3_RW, .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
2673       .resetfn = arm_cp_reset_ignore, .writefn = scr_write },
2674     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
2675       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
2676       .access = PL3_RW, .resetvalue = 0,
2677       .fieldoffset = offsetof(CPUARMState, cp15.sder) },
2678     { .name = "SDER",
2679       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
2680       .access = PL3_RW, .resetvalue = 0,
2681       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
2682       /* TODO: Implement NSACR trapping of secure EL1 accesses to EL3 */
2683     { .name = "NSACR", .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
2684       .access = PL3_W | PL1_R, .resetvalue = 0,
2685       .fieldoffset = offsetof(CPUARMState, cp15.nsacr) },
2686     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
2687       .access = PL3_RW, .writefn = vbar_write, .resetvalue = 0,
2688       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
2689     { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
2690       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
2691       .access = PL3_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
2692       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]) },
2693     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
2694       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
2695       .access = PL3_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
2696       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
2697     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
2698       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
2699       .access = PL3_RW, .writefn = vmsa_tcr_el1_write,
2700       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
2701       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
2702     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
2703       .type = ARM_CP_ALIAS,
2704       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
2705       .access = PL3_RW,
2706       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
2707     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
2708       .type = ARM_CP_ALIAS,
2709       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
2710       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
2711     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
2712       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
2713       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
2714     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
2715       .type = ARM_CP_ALIAS,
2716       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
2717       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[7]) },
2718     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
2719       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
2720       .access = PL3_RW, .writefn = vbar_write,
2721       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
2722       .resetvalue = 0 },
2723     { .name = "CPTR_EL3", .state = ARM_CP_STATE_AA64,
2724       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
2725       .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
2726       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
2727     REGINFO_SENTINEL
2728 };
2729
2730 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
2731 {
2732     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
2733      * but the AArch32 CTR has its own reginfo struct)
2734      */
2735     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
2736         return CP_ACCESS_TRAP;
2737     }
2738     return CP_ACCESS_OK;
2739 }
2740
2741 static const ARMCPRegInfo debug_cp_reginfo[] = {
2742     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
2743      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
2744      * unlike DBGDRAR it is never accessible from EL0.
2745      * DBGDSAR is deprecated and must RAZ from v8 anyway, so it has no AArch64
2746      * accessor.
2747      */
2748     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
2749       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2750     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
2751       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
2752       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2753     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
2754       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2755     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
2756     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
2757       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
2758       .access = PL1_RW,
2759       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
2760       .resetvalue = 0 },
2761     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
2762      * We don't implement the configurable EL0 access.
2763      */
2764     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
2765       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
2766       .type = ARM_CP_ALIAS,
2767       .access = PL1_R,
2768       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
2769       .resetfn = arm_cp_reset_ignore },
2770     /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
2771     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
2772       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
2773       .access = PL1_W, .type = ARM_CP_NOP },
2774     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
2775     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
2776       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
2777       .access = PL1_RW, .type = ARM_CP_NOP },
2778     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
2779      * implement vector catch debug events yet.
2780      */
2781     { .name = "DBGVCR",
2782       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
2783       .access = PL1_RW, .type = ARM_CP_NOP },
2784     REGINFO_SENTINEL
2785 };
2786
2787 static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
2788     /* 64 bit access versions of the (dummy) debug registers */
2789     { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
2790       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
2791     { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
2792       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
2793     REGINFO_SENTINEL
2794 };
2795
2796 void hw_watchpoint_update(ARMCPU *cpu, int n)
2797 {
2798     CPUARMState *env = &cpu->env;
2799     vaddr len = 0;
2800     vaddr wvr = env->cp15.dbgwvr[n];
2801     uint64_t wcr = env->cp15.dbgwcr[n];
2802     int mask;
2803     int flags = BP_CPU | BP_STOP_BEFORE_ACCESS;
2804
2805     if (env->cpu_watchpoint[n]) {
2806         cpu_watchpoint_remove_by_ref(CPU(cpu), env->cpu_watchpoint[n]);
2807         env->cpu_watchpoint[n] = NULL;
2808     }
2809
2810     if (!extract64(wcr, 0, 1)) {
2811         /* E bit clear : watchpoint disabled */
2812         return;
2813     }
2814
2815     switch (extract64(wcr, 3, 2)) {
2816     case 0:
2817         /* LSC 00 is reserved and must behave as if the wp is disabled */
2818         return;
2819     case 1:
2820         flags |= BP_MEM_READ;
2821         break;
2822     case 2:
2823         flags |= BP_MEM_WRITE;
2824         break;
2825     case 3:
2826         flags |= BP_MEM_ACCESS;
2827         break;
2828     }
2829
2830     /* Attempts to use both MASK and BAS fields simultaneously are
2831      * CONSTRAINED UNPREDICTABLE; we opt to ignore BAS in this case,
2832      * thus generating a watchpoint for every byte in the masked region.
2833      */
2834     mask = extract64(wcr, 24, 4);
2835     if (mask == 1 || mask == 2) {
2836         /* Reserved values of MASK; we must act as if the mask value was
2837          * some non-reserved value, or as if the watchpoint were disabled.
2838          * We choose the latter.
2839          */
2840         return;
2841     } else if (mask) {
2842         /* Watchpoint covers an aligned area up to 2GB in size */
2843         len = 1ULL << mask;
2844         /* If masked bits in WVR are not zero it's CONSTRAINED UNPREDICTABLE
2845          * whether the watchpoint fires when the unmasked bits match; we opt
2846          * to generate the exceptions.
2847          */
2848         wvr &= ~(len - 1);
2849     } else {
2850         /* Watchpoint covers bytes defined by the byte address select bits */
2851         int bas = extract64(wcr, 5, 8);
2852         int basstart;
2853
2854         if (bas == 0) {
2855             /* This must act as if the watchpoint is disabled */
2856             return;
2857         }
2858
2859         if (extract64(wvr, 2, 1)) {
2860             /* Deprecated case of an only 4-aligned address. BAS[7:4] are
2861              * ignored, and BAS[3:0] define which bytes to watch.
2862              */
2863             bas &= 0xf;
2864         }
2865         /* The BAS bits are supposed to be programmed to indicate a contiguous
2866          * range of bytes. Otherwise it is CONSTRAINED UNPREDICTABLE whether
2867          * we fire for each byte in the word/doubleword addressed by the WVR.
2868          * We choose to ignore any non-zero bits after the first range of 1s.
2869          */
2870         basstart = ctz32(bas);
2871         len = cto32(bas >> basstart);
2872         wvr += basstart;
2873     }
2874
2875     cpu_watchpoint_insert(CPU(cpu), wvr, len, flags,
2876                           &env->cpu_watchpoint[n]);
2877 }
2878
2879 void hw_watchpoint_update_all(ARMCPU *cpu)
2880 {
2881     int i;
2882     CPUARMState *env = &cpu->env;
2883
2884     /* Completely clear out existing QEMU watchpoints and our array, to
2885      * avoid possible stale entries following migration load.
2886      */
2887     cpu_watchpoint_remove_all(CPU(cpu), BP_CPU);
2888     memset(env->cpu_watchpoint, 0, sizeof(env->cpu_watchpoint));
2889
2890     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_watchpoint); i++) {
2891         hw_watchpoint_update(cpu, i);
2892     }
2893 }
2894
2895 static void dbgwvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2896                          uint64_t value)
2897 {
2898     ARMCPU *cpu = arm_env_get_cpu(env);
2899     int i = ri->crm;
2900
2901     /* Bits [63:49] are hardwired to the value of bit [48]; that is, the
2902      * register reads and behaves as if values written are sign extended.
2903      * Bits [1:0] are RES0.
2904      */
2905     value = sextract64(value, 0, 49) & ~3ULL;
2906
2907     raw_write(env, ri, value);
2908     hw_watchpoint_update(cpu, i);
2909 }
2910
2911 static void dbgwcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2912                          uint64_t value)
2913 {
2914     ARMCPU *cpu = arm_env_get_cpu(env);
2915     int i = ri->crm;
2916
2917     raw_write(env, ri, value);
2918     hw_watchpoint_update(cpu, i);
2919 }
2920
2921 void hw_breakpoint_update(ARMCPU *cpu, int n)
2922 {
2923     CPUARMState *env = &cpu->env;
2924     uint64_t bvr = env->cp15.dbgbvr[n];
2925     uint64_t bcr = env->cp15.dbgbcr[n];
2926     vaddr addr;
2927     int bt;
2928     int flags = BP_CPU;
2929
2930     if (env->cpu_breakpoint[n]) {
2931         cpu_breakpoint_remove_by_ref(CPU(cpu), env->cpu_breakpoint[n]);
2932         env->cpu_breakpoint[n] = NULL;
2933     }
2934
2935     if (!extract64(bcr, 0, 1)) {
2936         /* E bit clear : watchpoint disabled */
2937         return;
2938     }
2939
2940     bt = extract64(bcr, 20, 4);
2941
2942     switch (bt) {
2943     case 4: /* unlinked address mismatch (reserved if AArch64) */
2944     case 5: /* linked address mismatch (reserved if AArch64) */
2945         qemu_log_mask(LOG_UNIMP,
2946                       "arm: address mismatch breakpoint types not implemented");
2947         return;
2948     case 0: /* unlinked address match */
2949     case 1: /* linked address match */
2950     {
2951         /* Bits [63:49] are hardwired to the value of bit [48]; that is,
2952          * we behave as if the register was sign extended. Bits [1:0] are
2953          * RES0. The BAS field is used to allow setting breakpoints on 16
2954          * bit wide instructions; it is CONSTRAINED UNPREDICTABLE whether
2955          * a bp will fire if the addresses covered by the bp and the addresses
2956          * covered by the insn overlap but the insn doesn't start at the
2957          * start of the bp address range. We choose to require the insn and
2958          * the bp to have the same address. The constraints on writing to
2959          * BAS enforced in dbgbcr_write mean we have only four cases:
2960          *  0b0000  => no breakpoint
2961          *  0b0011  => breakpoint on addr
2962          *  0b1100  => breakpoint on addr + 2
2963          *  0b1111  => breakpoint on addr
2964          * See also figure D2-3 in the v8 ARM ARM (DDI0487A.c).
2965          */
2966         int bas = extract64(bcr, 5, 4);
2967         addr = sextract64(bvr, 0, 49) & ~3ULL;
2968         if (bas == 0) {
2969             return;
2970         }
2971         if (bas == 0xc) {
2972             addr += 2;
2973         }
2974         break;
2975     }
2976     case 2: /* unlinked context ID match */
2977     case 8: /* unlinked VMID match (reserved if no EL2) */
2978     case 10: /* unlinked context ID and VMID match (reserved if no EL2) */
2979         qemu_log_mask(LOG_UNIMP,
2980                       "arm: unlinked context breakpoint types not implemented");
2981         return;
2982     case 9: /* linked VMID match (reserved if no EL2) */
2983     case 11: /* linked context ID and VMID match (reserved if no EL2) */
2984     case 3: /* linked context ID match */
2985     default:
2986         /* We must generate no events for Linked context matches (unless
2987          * they are linked to by some other bp/wp, which is handled in
2988          * updates for the linking bp/wp). We choose to also generate no events
2989          * for reserved values.
2990          */
2991         return;
2992     }
2993
2994     cpu_breakpoint_insert(CPU(cpu), addr, flags, &env->cpu_breakpoint[n]);
2995 }
2996
2997 void hw_breakpoint_update_all(ARMCPU *cpu)
2998 {
2999     int i;
3000     CPUARMState *env = &cpu->env;
3001
3002     /* Completely clear out existing QEMU breakpoints and our array, to
3003      * avoid possible stale entries following migration load.
3004      */
3005     cpu_breakpoint_remove_all(CPU(cpu), BP_CPU);
3006     memset(env->cpu_breakpoint, 0, sizeof(env->cpu_breakpoint));
3007
3008     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_breakpoint); i++) {
3009         hw_breakpoint_update(cpu, i);
3010     }
3011 }
3012
3013 static void dbgbvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3014                          uint64_t value)
3015 {
3016     ARMCPU *cpu = arm_env_get_cpu(env);
3017     int i = ri->crm;
3018
3019     raw_write(env, ri, value);
3020     hw_breakpoint_update(cpu, i);
3021 }
3022
3023 static void dbgbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3024                          uint64_t value)
3025 {
3026     ARMCPU *cpu = arm_env_get_cpu(env);
3027     int i = ri->crm;
3028
3029     /* BAS[3] is a read-only copy of BAS[2], and BAS[1] a read-only
3030      * copy of BAS[0].
3031      */
3032     value = deposit64(value, 6, 1, extract64(value, 5, 1));
3033     value = deposit64(value, 8, 1, extract64(value, 7, 1));
3034
3035     raw_write(env, ri, value);
3036     hw_breakpoint_update(cpu, i);
3037 }
3038
3039 static void define_debug_regs(ARMCPU *cpu)
3040 {
3041     /* Define v7 and v8 architectural debug registers.
3042      * These are just dummy implementations for now.
3043      */
3044     int i;
3045     int wrps, brps, ctx_cmps;
3046     ARMCPRegInfo dbgdidr = {
3047         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
3048         .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
3049     };
3050
3051     /* Note that all these register fields hold "number of Xs minus 1". */
3052     brps = extract32(cpu->dbgdidr, 24, 4);
3053     wrps = extract32(cpu->dbgdidr, 28, 4);
3054     ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
3055
3056     assert(ctx_cmps <= brps);
3057
3058     /* The DBGDIDR and ID_AA64DFR0_EL1 define various properties
3059      * of the debug registers such as number of breakpoints;
3060      * check that if they both exist then they agree.
3061      */
3062     if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
3063         assert(extract32(cpu->id_aa64dfr0, 12, 4) == brps);
3064         assert(extract32(cpu->id_aa64dfr0, 20, 4) == wrps);
3065         assert(extract32(cpu->id_aa64dfr0, 28, 4) == ctx_cmps);
3066     }
3067
3068     define_one_arm_cp_reg(cpu, &dbgdidr);
3069     define_arm_cp_regs(cpu, debug_cp_reginfo);
3070
3071     if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
3072         define_arm_cp_regs(cpu, debug_lpae_cp_reginfo);
3073     }
3074
3075     for (i = 0; i < brps + 1; i++) {
3076         ARMCPRegInfo dbgregs[] = {
3077             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
3078               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
3079               .access = PL1_RW,
3080               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
3081               .writefn = dbgbvr_write, .raw_writefn = raw_write
3082             },
3083             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
3084               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
3085               .access = PL1_RW,
3086               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
3087               .writefn = dbgbcr_write, .raw_writefn = raw_write
3088             },
3089             REGINFO_SENTINEL
3090         };
3091         define_arm_cp_regs(cpu, dbgregs);
3092     }
3093
3094     for (i = 0; i < wrps + 1; i++) {
3095         ARMCPRegInfo dbgregs[] = {
3096             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
3097               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
3098               .access = PL1_RW,
3099               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
3100               .writefn = dbgwvr_write, .raw_writefn = raw_write
3101             },
3102             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
3103               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
3104               .access = PL1_RW,
3105               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
3106               .writefn = dbgwcr_write, .raw_writefn = raw_write
3107             },
3108             REGINFO_SENTINEL
3109         };
3110         define_arm_cp_regs(cpu, dbgregs);
3111     }
3112 }
3113
3114 void register_cp_regs_for_features(ARMCPU *cpu)
3115 {
3116     /* Register all the coprocessor registers based on feature bits */
3117     CPUARMState *env = &cpu->env;
3118     if (arm_feature(env, ARM_FEATURE_M)) {
3119         /* M profile has no coprocessor registers */
3120         return;
3121     }
3122
3123     define_arm_cp_regs(cpu, cp_reginfo);
3124     if (!arm_feature(env, ARM_FEATURE_V8)) {
3125         /* Must go early as it is full of wildcards that may be
3126          * overridden by later definitions.
3127          */
3128         define_arm_cp_regs(cpu, not_v8_cp_reginfo);
3129     }
3130
3131     if (arm_feature(env, ARM_FEATURE_V6)) {
3132         /* The ID registers all have impdef reset values */
3133         ARMCPRegInfo v6_idregs[] = {
3134             { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
3135               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
3136               .access = PL1_R, .type = ARM_CP_CONST,
3137               .resetvalue = cpu->id_pfr0 },
3138             { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
3139               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
3140               .access = PL1_R, .type = ARM_CP_CONST,
3141               .resetvalue = cpu->id_pfr1 },
3142             { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
3143               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
3144               .access = PL1_R, .type = ARM_CP_CONST,
3145               .resetvalue = cpu->id_dfr0 },
3146             { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
3147               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
3148               .access = PL1_R, .type = ARM_CP_CONST,
3149               .resetvalue = cpu->id_afr0 },
3150             { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
3151               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
3152               .access = PL1_R, .type = ARM_CP_CONST,
3153               .resetvalue = cpu->id_mmfr0 },
3154             { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
3155               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
3156               .access = PL1_R, .type = ARM_CP_CONST,
3157               .resetvalue = cpu->id_mmfr1 },
3158             { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
3159               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
3160               .access = PL1_R, .type = ARM_CP_CONST,
3161               .resetvalue = cpu->id_mmfr2 },
3162             { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
3163               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
3164               .access = PL1_R, .type = ARM_CP_CONST,
3165               .resetvalue = cpu->id_mmfr3 },
3166             { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
3167               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
3168               .access = PL1_R, .type = ARM_CP_CONST,
3169               .resetvalue = cpu->id_isar0 },
3170             { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
3171               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
3172               .access = PL1_R, .type = ARM_CP_CONST,
3173               .resetvalue = cpu->id_isar1 },
3174             { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
3175               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
3176               .access = PL1_R, .type = ARM_CP_CONST,
3177               .resetvalue = cpu->id_isar2 },
3178             { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
3179               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
3180               .access = PL1_R, .type = ARM_CP_CONST,
3181               .resetvalue = cpu->id_isar3 },
3182             { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
3183               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
3184               .access = PL1_R, .type = ARM_CP_CONST,
3185               .resetvalue = cpu->id_isar4 },
3186             { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
3187               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
3188               .access = PL1_R, .type = ARM_CP_CONST,
3189               .resetvalue = cpu->id_isar5 },
3190             /* 6..7 are as yet unallocated and must RAZ */
3191             { .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
3192               .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
3193               .resetvalue = 0 },
3194             { .name = "ID_ISAR7", .cp = 15, .crn = 0, .crm = 2,
3195               .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
3196               .resetvalue = 0 },
3197             REGINFO_SENTINEL
3198         };
3199         define_arm_cp_regs(cpu, v6_idregs);
3200         define_arm_cp_regs(cpu, v6_cp_reginfo);
3201     } else {
3202         define_arm_cp_regs(cpu, not_v6_cp_reginfo);
3203     }
3204     if (arm_feature(env, ARM_FEATURE_V6K)) {
3205         define_arm_cp_regs(cpu, v6k_cp_reginfo);
3206     }
3207     if (arm_feature(env, ARM_FEATURE_V7MP)) {
3208         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
3209     }
3210     if (arm_feature(env, ARM_FEATURE_V7)) {
3211         /* v7 performance monitor control register: same implementor
3212          * field as main ID register, and we implement only the cycle
3213          * count register.
3214          */
3215 #ifndef CONFIG_USER_ONLY
3216         ARMCPRegInfo pmcr = {
3217             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
3218             .access = PL0_RW,
3219             .type = ARM_CP_IO | ARM_CP_ALIAS,
3220             .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
3221             .accessfn = pmreg_access, .writefn = pmcr_write,
3222             .raw_writefn = raw_write,
3223         };
3224         ARMCPRegInfo pmcr64 = {
3225             .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
3226             .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
3227             .access = PL0_RW, .accessfn = pmreg_access,
3228             .type = ARM_CP_IO,
3229             .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
3230             .resetvalue = cpu->midr & 0xff000000,
3231             .writefn = pmcr_write, .raw_writefn = raw_write,
3232         };
3233         define_one_arm_cp_reg(cpu, &pmcr);
3234         define_one_arm_cp_reg(cpu, &pmcr64);
3235 #endif
3236         ARMCPRegInfo clidr = {
3237             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
3238             .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
3239             .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->clidr
3240         };
3241         define_one_arm_cp_reg(cpu, &clidr);
3242         define_arm_cp_regs(cpu, v7_cp_reginfo);
3243         define_debug_regs(cpu);
3244     } else {
3245         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
3246     }
3247     if (arm_feature(env, ARM_FEATURE_V8)) {
3248         /* AArch64 ID registers, which all have impdef reset values */
3249         ARMCPRegInfo v8_idregs[] = {
3250             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
3251               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
3252               .access = PL1_R, .type = ARM_CP_CONST,
3253               .resetvalue = cpu->id_aa64pfr0 },
3254             { .name = "ID_AA64PFR1_EL1", .state = ARM_CP_STATE_AA64,
3255               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
3256               .access = PL1_R, .type = ARM_CP_CONST,
3257               .resetvalue = cpu->id_aa64pfr1},
3258             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
3259               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
3260               .access = PL1_R, .type = ARM_CP_CONST,
3261               /* We mask out the PMUVer field, because we don't currently
3262                * implement the PMU. Not advertising it prevents the guest
3263                * from trying to use it and getting UNDEFs on registers we
3264                * don't implement.
3265                */
3266               .resetvalue = cpu->id_aa64dfr0 & ~0xf00 },
3267             { .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
3268               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
3269               .access = PL1_R, .type = ARM_CP_CONST,
3270               .resetvalue = cpu->id_aa64dfr1 },
3271             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
3272               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
3273               .access = PL1_R, .type = ARM_CP_CONST,
3274               .resetvalue = cpu->id_aa64afr0 },
3275             { .name = "ID_AA64AFR1_EL1", .state = ARM_CP_STATE_AA64,
3276               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
3277               .access = PL1_R, .type = ARM_CP_CONST,
3278               .resetvalue = cpu->id_aa64afr1 },
3279             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
3280               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
3281               .access = PL1_R, .type = ARM_CP_CONST,
3282               .resetvalue = cpu->id_aa64isar0 },
3283             { .name = "ID_AA64ISAR1_EL1", .state = ARM_CP_STATE_AA64,
3284               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
3285               .access = PL1_R, .type = ARM_CP_CONST,
3286               .resetvalue = cpu->id_aa64isar1 },
3287             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
3288               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
3289               .access = PL1_R, .type = ARM_CP_CONST,
3290               .resetvalue = cpu->id_aa64mmfr0 },
3291             { .name = "ID_AA64MMFR1_EL1", .state = ARM_CP_STATE_AA64,
3292               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
3293               .access = PL1_R, .type = ARM_CP_CONST,
3294               .resetvalue = cpu->id_aa64mmfr1 },
3295             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
3296               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
3297               .access = PL1_R, .type = ARM_CP_CONST,
3298               .resetvalue = cpu->mvfr0 },
3299             { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
3300               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
3301               .access = PL1_R, .type = ARM_CP_CONST,
3302               .resetvalue = cpu->mvfr1 },
3303             { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
3304               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
3305               .access = PL1_R, .type = ARM_CP_CONST,
3306               .resetvalue = cpu->mvfr2 },
3307             REGINFO_SENTINEL
3308         };
3309         /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
3310         if (!arm_feature(env, ARM_FEATURE_EL3) &&
3311             !arm_feature(env, ARM_FEATURE_EL2)) {
3312             ARMCPRegInfo rvbar = {
3313                 .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
3314                 .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
3315                 .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
3316             };
3317             define_one_arm_cp_reg(cpu, &rvbar);
3318         }
3319         define_arm_cp_regs(cpu, v8_idregs);
3320         define_arm_cp_regs(cpu, v8_cp_reginfo);
3321     }
3322     if (arm_feature(env, ARM_FEATURE_EL2)) {
3323         define_arm_cp_regs(cpu, v8_el2_cp_reginfo);
3324         /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
3325         if (!arm_feature(env, ARM_FEATURE_EL3)) {
3326             ARMCPRegInfo rvbar = {
3327                 .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
3328                 .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
3329                 .type = ARM_CP_CONST, .access = PL2_R, .resetvalue = cpu->rvbar
3330             };
3331             define_one_arm_cp_reg(cpu, &rvbar);
3332         }
3333     } else {
3334         /* If EL2 is missing but higher ELs are enabled, we need to
3335          * register the no_el2 reginfos.
3336          */
3337         if (arm_feature(env, ARM_FEATURE_EL3)) {
3338             define_arm_cp_regs(cpu, v8_el3_no_el2_cp_reginfo);
3339         }
3340     }
3341     if (arm_feature(env, ARM_FEATURE_EL3)) {
3342         define_arm_cp_regs(cpu, el3_cp_reginfo);
3343         ARMCPRegInfo rvbar = {
3344             .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
3345             .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
3346             .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar
3347         };
3348         define_one_arm_cp_reg(cpu, &rvbar);
3349     }
3350     if (arm_feature(env, ARM_FEATURE_MPU)) {
3351         /* These are the MPU registers prior to PMSAv6. Any new
3352          * PMSA core later than the ARM946 will require that we
3353          * implement the PMSAv6 or PMSAv7 registers, which are
3354          * completely different.
3355          */
3356         assert(!arm_feature(env, ARM_FEATURE_V6));
3357         define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
3358     } else {
3359         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
3360     }
3361     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
3362         define_arm_cp_regs(cpu, t2ee_cp_reginfo);
3363     }
3364     if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
3365         define_arm_cp_regs(cpu, generic_timer_cp_reginfo);
3366     }
3367     if (arm_feature(env, ARM_FEATURE_VAPA)) {
3368         define_arm_cp_regs(cpu, vapa_cp_reginfo);
3369     }
3370     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
3371         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
3372     }
3373     if (arm_feature(env, ARM_FEATURE_CACHE_DIRTY_REG)) {
3374         define_arm_cp_regs(cpu, cache_dirty_status_cp_reginfo);
3375     }
3376     if (arm_feature(env, ARM_FEATURE_CACHE_BLOCK_OPS)) {
3377         define_arm_cp_regs(cpu, cache_block_ops_cp_reginfo);
3378     }
3379     if (arm_feature(env, ARM_FEATURE_OMAPCP)) {
3380         define_arm_cp_regs(cpu, omap_cp_reginfo);
3381     }
3382     if (arm_feature(env, ARM_FEATURE_STRONGARM)) {
3383         define_arm_cp_regs(cpu, strongarm_cp_reginfo);
3384     }
3385     if (arm_feature(env, ARM_FEATURE_XSCALE)) {
3386         define_arm_cp_regs(cpu, xscale_cp_reginfo);
3387     }
3388     if (arm_feature(env, ARM_FEATURE_DUMMY_C15_REGS)) {
3389         define_arm_cp_regs(cpu, dummy_c15_cp_reginfo);
3390     }
3391     if (arm_feature(env, ARM_FEATURE_LPAE)) {
3392         define_arm_cp_regs(cpu, lpae_cp_reginfo);
3393     }
3394     /* Slightly awkwardly, the OMAP and StrongARM cores need all of
3395      * cp15 crn=0 to be writes-ignored, whereas for other cores they should
3396      * be read-only (ie write causes UNDEF exception).
3397      */
3398     {
3399         ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
3400             /* Pre-v8 MIDR space.
3401              * Note that the MIDR isn't a simple constant register because
3402              * of the TI925 behaviour where writes to another register can
3403              * cause the MIDR value to change.
3404              *
3405              * Unimplemented registers in the c15 0 0 0 space default to
3406              * MIDR. Define MIDR first as this entire space, then CTR, TCMTR
3407              * and friends override accordingly.
3408              */
3409             { .name = "MIDR",
3410               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
3411               .access = PL1_R, .resetvalue = cpu->midr,
3412               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
3413               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
3414               .type = ARM_CP_OVERRIDE },
3415             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
3416             { .name = "DUMMY",
3417               .cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
3418               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3419             { .name = "DUMMY",
3420               .cp = 15, .crn = 0, .crm = 4, .opc1 = 0, .opc2 = CP_ANY,
3421               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3422             { .name = "DUMMY",
3423               .cp = 15, .crn = 0, .crm = 5, .opc1 = 0, .opc2 = CP_ANY,
3424               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3425             { .name = "DUMMY",
3426               .cp = 15, .crn = 0, .crm = 6, .opc1 = 0, .opc2 = CP_ANY,
3427               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3428             { .name = "DUMMY",
3429               .cp = 15, .crn = 0, .crm = 7, .opc1 = 0, .opc2 = CP_ANY,
3430               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3431             REGINFO_SENTINEL
3432         };
3433         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
3434             /* v8 MIDR -- the wildcard isn't necessary, and nor is the
3435              * variable-MIDR TI925 behaviour. Instead we have a single
3436              * (strictly speaking IMPDEF) alias of the MIDR, REVIDR.
3437              */
3438             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
3439               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
3440               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
3441             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
3442               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
3443               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
3444             REGINFO_SENTINEL
3445         };
3446         ARMCPRegInfo id_cp_reginfo[] = {
3447             /* These are common to v8 and pre-v8 */
3448             { .name = "CTR",
3449               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
3450               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
3451             { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
3452               .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
3453               .access = PL0_R, .accessfn = ctr_el0_access,
3454               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
3455             /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
3456             { .name = "TCMTR",
3457               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
3458               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3459             { .name = "TLBTR",
3460               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
3461               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3462             REGINFO_SENTINEL
3463         };
3464         ARMCPRegInfo crn0_wi_reginfo = {
3465             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
3466             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
3467             .type = ARM_CP_NOP | ARM_CP_OVERRIDE
3468         };
3469         if (arm_feature(env, ARM_FEATURE_OMAPCP) ||
3470             arm_feature(env, ARM_FEATURE_STRONGARM)) {
3471             ARMCPRegInfo *r;
3472             /* Register the blanket "writes ignored" value first to cover the
3473              * whole space. Then update the specific ID registers to allow write
3474              * access, so that they ignore writes rather than causing them to
3475              * UNDEF.
3476              */
3477             define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
3478             for (r = id_pre_v8_midr_cp_reginfo;
3479                  r->type != ARM_CP_SENTINEL; r++) {
3480                 r->access = PL1_RW;
3481             }
3482             for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
3483                 r->access = PL1_RW;
3484             }
3485         }
3486         if (arm_feature(env, ARM_FEATURE_V8)) {
3487             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
3488         } else {
3489             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
3490         }
3491         define_arm_cp_regs(cpu, id_cp_reginfo);
3492     }
3493
3494     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
3495         define_arm_cp_regs(cpu, mpidr_cp_reginfo);
3496     }
3497
3498     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
3499         ARMCPRegInfo auxcr = {
3500             .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
3501             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
3502             .access = PL1_RW, .type = ARM_CP_CONST,
3503             .resetvalue = cpu->reset_auxcr
3504         };
3505         define_one_arm_cp_reg(cpu, &auxcr);
3506     }
3507
3508     if (arm_feature(env, ARM_FEATURE_CBAR)) {
3509         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
3510             /* 32 bit view is [31:18] 0...0 [43:32]. */
3511             uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
3512                 | extract64(cpu->reset_cbar, 32, 12);
3513             ARMCPRegInfo cbar_reginfo[] = {
3514                 { .name = "CBAR",
3515                   .type = ARM_CP_CONST,
3516                   .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
3517                   .access = PL1_R, .resetvalue = cpu->reset_cbar },
3518                 { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
3519                   .type = ARM_CP_CONST,
3520                   .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
3521                   .access = PL1_R, .resetvalue = cbar32 },
3522                 REGINFO_SENTINEL
3523             };
3524             /* We don't implement a r/w 64 bit CBAR currently */
3525             assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
3526             define_arm_cp_regs(cpu, cbar_reginfo);
3527         } else {
3528             ARMCPRegInfo cbar = {
3529                 .name = "CBAR",
3530                 .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
3531                 .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
3532                 .fieldoffset = offsetof(CPUARMState,
3533                                         cp15.c15_config_base_address)
3534             };
3535             if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
3536                 cbar.access = PL1_R;
3537                 cbar.fieldoffset = 0;
3538                 cbar.type = ARM_CP_CONST;
3539             }
3540             define_one_arm_cp_reg(cpu, &cbar);
3541         }
3542     }
3543
3544     /* Generic registers whose values depend on the implementation */
3545     {
3546         ARMCPRegInfo sctlr = {
3547             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
3548             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
3549             .access = PL1_RW,
3550             .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
3551                                    offsetof(CPUARMState, cp15.sctlr_ns) },
3552             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
3553             .raw_writefn = raw_write,
3554         };
3555         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
3556             /* Normally we would always end the TB on an SCTLR write, but Linux
3557              * arch/arm/mach-pxa/sleep.S expects two instructions following
3558              * an MMU enable to execute from cache.  Imitate this behaviour.
3559              */
3560             sctlr.type |= ARM_CP_SUPPRESS_TB_END;
3561         }
3562         define_one_arm_cp_reg(cpu, &sctlr);
3563     }
3564 }
3565
3566 ARMCPU *cpu_arm_init(const char *cpu_model)
3567 {
3568     return ARM_CPU(cpu_generic_init(TYPE_ARM_CPU, cpu_model));
3569 }
3570
3571 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
3572 {
3573     CPUState *cs = CPU(cpu);
3574     CPUARMState *env = &cpu->env;
3575
3576     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
3577         gdb_register_coprocessor(cs, aarch64_fpu_gdb_get_reg,
3578                                  aarch64_fpu_gdb_set_reg,
3579                                  34, "aarch64-fpu.xml", 0);
3580     } else if (arm_feature(env, ARM_FEATURE_NEON)) {
3581         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3582                                  51, "arm-neon.xml", 0);
3583     } else if (arm_feature(env, ARM_FEATURE_VFP3)) {
3584         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3585                                  35, "arm-vfp3.xml", 0);
3586     } else if (arm_feature(env, ARM_FEATURE_VFP)) {
3587         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3588                                  19, "arm-vfp.xml", 0);
3589     }
3590 }
3591
3592 /* Sort alphabetically by type name, except for "any". */
3593 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
3594 {
3595     ObjectClass *class_a = (ObjectClass *)a;
3596     ObjectClass *class_b = (ObjectClass *)b;
3597     const char *name_a, *name_b;
3598
3599     name_a = object_class_get_name(class_a);
3600     name_b = object_class_get_name(class_b);
3601     if (strcmp(name_a, "any-" TYPE_ARM_CPU) == 0) {
3602         return 1;
3603     } else if (strcmp(name_b, "any-" TYPE_ARM_CPU) == 0) {
3604         return -1;
3605     } else {
3606         return strcmp(name_a, name_b);
3607     }
3608 }
3609
3610 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
3611 {
3612     ObjectClass *oc = data;
3613     CPUListState *s = user_data;
3614     const char *typename;
3615     char *name;
3616
3617     typename = object_class_get_name(oc);
3618     name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_ARM_CPU));
3619     (*s->cpu_fprintf)(s->file, "  %s\n",
3620                       name);
3621     g_free(name);
3622 }
3623
3624 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
3625 {
3626     CPUListState s = {
3627         .file = f,
3628         .cpu_fprintf = cpu_fprintf,
3629     };
3630     GSList *list;
3631
3632     list = object_class_get_list(TYPE_ARM_CPU, false);
3633     list = g_slist_sort(list, arm_cpu_list_compare);
3634     (*cpu_fprintf)(f, "Available CPUs:\n");
3635     g_slist_foreach(list, arm_cpu_list_entry, &s);
3636     g_slist_free(list);
3637 #ifdef CONFIG_KVM
3638     /* The 'host' CPU type is dynamically registered only if KVM is
3639      * enabled, so we have to special-case it here:
3640      */
3641     (*cpu_fprintf)(f, "  host (only available in KVM mode)\n");
3642 #endif
3643 }
3644
3645 static void arm_cpu_add_definition(gpointer data, gpointer user_data)
3646 {
3647     ObjectClass *oc = data;
3648     CpuDefinitionInfoList **cpu_list = user_data;
3649     CpuDefinitionInfoList *entry;
3650     CpuDefinitionInfo *info;
3651     const char *typename;
3652
3653     typename = object_class_get_name(oc);
3654     info = g_malloc0(sizeof(*info));
3655     info->name = g_strndup(typename,
3656                            strlen(typename) - strlen("-" TYPE_ARM_CPU));
3657
3658     entry = g_malloc0(sizeof(*entry));
3659     entry->value = info;
3660     entry->next = *cpu_list;
3661     *cpu_list = entry;
3662 }
3663
3664 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
3665 {
3666     CpuDefinitionInfoList *cpu_list = NULL;
3667     GSList *list;
3668
3669     list = object_class_get_list(TYPE_ARM_CPU, false);
3670     g_slist_foreach(list, arm_cpu_add_definition, &cpu_list);
3671     g_slist_free(list);
3672
3673     return cpu_list;
3674 }
3675
3676 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
3677                                    void *opaque, int state, int secstate,
3678                                    int crm, int opc1, int opc2)
3679 {
3680     /* Private utility function for define_one_arm_cp_reg_with_opaque():
3681      * add a single reginfo struct to the hash table.
3682      */
3683     uint32_t *key = g_new(uint32_t, 1);
3684     ARMCPRegInfo *r2 = g_memdup(r, sizeof(ARMCPRegInfo));
3685     int is64 = (r->type & ARM_CP_64BIT) ? 1 : 0;
3686     int ns = (secstate & ARM_CP_SECSTATE_NS) ? 1 : 0;
3687
3688     /* Reset the secure state to the specific incoming state.  This is
3689      * necessary as the register may have been defined with both states.
3690      */
3691     r2->secure = secstate;
3692
3693     if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
3694         /* Register is banked (using both entries in array).
3695          * Overwriting fieldoffset as the array is only used to define
3696          * banked registers but later only fieldoffset is used.
3697          */
3698         r2->fieldoffset = r->bank_fieldoffsets[ns];
3699     }
3700
3701     if (state == ARM_CP_STATE_AA32) {
3702         if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
3703             /* If the register is banked then we don't need to migrate or
3704              * reset the 32-bit instance in certain cases:
3705              *
3706              * 1) If the register has both 32-bit and 64-bit instances then we
3707              *    can count on the 64-bit instance taking care of the
3708              *    non-secure bank.
3709              * 2) If ARMv8 is enabled then we can count on a 64-bit version
3710              *    taking care of the secure bank.  This requires that separate
3711              *    32 and 64-bit definitions are provided.
3712              */
3713             if ((r->state == ARM_CP_STATE_BOTH && ns) ||
3714                 (arm_feature(&cpu->env, ARM_FEATURE_V8) && !ns)) {
3715                 r2->type |= ARM_CP_ALIAS;
3716                 r2->resetfn = arm_cp_reset_ignore;
3717             }
3718         } else if ((secstate != r->secure) && !ns) {
3719             /* The register is not banked so we only want to allow migration of
3720              * the non-secure instance.
3721              */
3722             r2->type |= ARM_CP_ALIAS;
3723             r2->resetfn = arm_cp_reset_ignore;
3724         }
3725
3726         if (r->state == ARM_CP_STATE_BOTH) {
3727             /* We assume it is a cp15 register if the .cp field is left unset.
3728              */
3729             if (r2->cp == 0) {
3730                 r2->cp = 15;
3731             }
3732
3733 #ifdef HOST_WORDS_BIGENDIAN
3734             if (r2->fieldoffset) {
3735                 r2->fieldoffset += sizeof(uint32_t);
3736             }
3737 #endif
3738         }
3739     }
3740     if (state == ARM_CP_STATE_AA64) {
3741         /* To allow abbreviation of ARMCPRegInfo
3742          * definitions, we treat cp == 0 as equivalent to
3743          * the value for "standard guest-visible sysreg".
3744          * STATE_BOTH definitions are also always "standard
3745          * sysreg" in their AArch64 view (the .cp value may
3746          * be non-zero for the benefit of the AArch32 view).
3747          */
3748         if (r->cp == 0 || r->state == ARM_CP_STATE_BOTH) {
3749             r2->cp = CP_REG_ARM64_SYSREG_CP;
3750         }
3751         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
3752                                   r2->opc0, opc1, opc2);
3753     } else {
3754         *key = ENCODE_CP_REG(r2->cp, is64, ns, r2->crn, crm, opc1, opc2);
3755     }
3756     if (opaque) {
3757         r2->opaque = opaque;
3758     }
3759     /* reginfo passed to helpers is correct for the actual access,
3760      * and is never ARM_CP_STATE_BOTH:
3761      */
3762     r2->state = state;
3763     /* Make sure reginfo passed to helpers for wildcarded regs
3764      * has the correct crm/opc1/opc2 for this reg, not CP_ANY:
3765      */
3766     r2->crm = crm;
3767     r2->opc1 = opc1;
3768     r2->opc2 = opc2;
3769     /* By convention, for wildcarded registers only the first
3770      * entry is used for migration; the others are marked as
3771      * ALIAS so we don't try to transfer the register
3772      * multiple times. Special registers (ie NOP/WFI) are
3773      * never migratable and not even raw-accessible.
3774      */
3775     if ((r->type & ARM_CP_SPECIAL)) {
3776         r2->type |= ARM_CP_NO_RAW;
3777     }
3778     if (((r->crm == CP_ANY) && crm != 0) ||
3779         ((r->opc1 == CP_ANY) && opc1 != 0) ||
3780         ((r->opc2 == CP_ANY) && opc2 != 0)) {
3781         r2->type |= ARM_CP_ALIAS;
3782     }
3783
3784     /* Check that raw accesses are either forbidden or handled. Note that
3785      * we can't assert this earlier because the setup of fieldoffset for
3786      * banked registers has to be done first.
3787      */
3788     if (!(r2->type & ARM_CP_NO_RAW)) {
3789         assert(!raw_accessors_invalid(r2));
3790     }
3791
3792     /* Overriding of an existing definition must be explicitly
3793      * requested.
3794      */
3795     if (!(r->type & ARM_CP_OVERRIDE)) {
3796         ARMCPRegInfo *oldreg;
3797         oldreg = g_hash_table_lookup(cpu->cp_regs, key);
3798         if (oldreg && !(oldreg->type & ARM_CP_OVERRIDE)) {
3799             fprintf(stderr, "Register redefined: cp=%d %d bit "
3800                     "crn=%d crm=%d opc1=%d opc2=%d, "
3801                     "was %s, now %s\n", r2->cp, 32 + 32 * is64,
3802                     r2->crn, r2->crm, r2->opc1, r2->opc2,
3803                     oldreg->name, r2->name);
3804             g_assert_not_reached();
3805         }
3806     }
3807     g_hash_table_insert(cpu->cp_regs, key, r2);
3808 }
3809
3810
3811 void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
3812                                        const ARMCPRegInfo *r, void *opaque)
3813 {
3814     /* Define implementations of coprocessor registers.
3815      * We store these in a hashtable because typically
3816      * there are less than 150 registers in a space which
3817      * is 16*16*16*8*8 = 262144 in size.
3818      * Wildcarding is supported for the crm, opc1 and opc2 fields.
3819      * If a register is defined twice then the second definition is
3820      * used, so this can be used to define some generic registers and
3821      * then override them with implementation specific variations.
3822      * At least one of the original and the second definition should
3823      * include ARM_CP_OVERRIDE in its type bits -- this is just a guard
3824      * against accidental use.
3825      *
3826      * The state field defines whether the register is to be
3827      * visible in the AArch32 or AArch64 execution state. If the
3828      * state is set to ARM_CP_STATE_BOTH then we synthesise a
3829      * reginfo structure for the AArch32 view, which sees the lower
3830      * 32 bits of the 64 bit register.
3831      *
3832      * Only registers visible in AArch64 may set r->opc0; opc0 cannot
3833      * be wildcarded. AArch64 registers are always considered to be 64
3834      * bits; the ARM_CP_64BIT* flag applies only to the AArch32 view of
3835      * the register, if any.
3836      */
3837     int crm, opc1, opc2, state;
3838     int crmmin = (r->crm == CP_ANY) ? 0 : r->crm;
3839     int crmmax = (r->crm == CP_ANY) ? 15 : r->crm;
3840     int opc1min = (r->opc1 == CP_ANY) ? 0 : r->opc1;
3841     int opc1max = (r->opc1 == CP_ANY) ? 7 : r->opc1;
3842     int opc2min = (r->opc2 == CP_ANY) ? 0 : r->opc2;
3843     int opc2max = (r->opc2 == CP_ANY) ? 7 : r->opc2;
3844     /* 64 bit registers have only CRm and Opc1 fields */
3845     assert(!((r->type & ARM_CP_64BIT) && (r->opc2 || r->crn)));
3846     /* op0 only exists in the AArch64 encodings */
3847     assert((r->state != ARM_CP_STATE_AA32) || (r->opc0 == 0));
3848     /* AArch64 regs are all 64 bit so ARM_CP_64BIT is meaningless */
3849     assert((r->state != ARM_CP_STATE_AA64) || !(r->type & ARM_CP_64BIT));
3850     /* The AArch64 pseudocode CheckSystemAccess() specifies that op1
3851      * encodes a minimum access level for the register. We roll this
3852      * runtime check into our general permission check code, so check
3853      * here that the reginfo's specified permissions are strict enough
3854      * to encompass the generic architectural permission check.
3855      */
3856     if (r->state != ARM_CP_STATE_AA32) {
3857         int mask = 0;
3858         switch (r->opc1) {
3859         case 0: case 1: case 2:
3860             /* min_EL EL1 */
3861             mask = PL1_RW;
3862             break;
3863         case 3:
3864             /* min_EL EL0 */
3865             mask = PL0_RW;
3866             break;
3867         case 4:
3868             /* min_EL EL2 */
3869             mask = PL2_RW;
3870             break;
3871         case 5:
3872             /* unallocated encoding, so not possible */
3873             assert(false);
3874             break;
3875         case 6:
3876             /* min_EL EL3 */
3877             mask = PL3_RW;
3878             break;
3879         case 7:
3880             /* min_EL EL1, secure mode only (we don't check the latter) */
3881             mask = PL1_RW;
3882             break;
3883         default:
3884             /* broken reginfo with out-of-range opc1 */
3885             assert(false);
3886             break;
3887         }
3888         /* assert our permissions are not too lax (stricter is fine) */
3889         assert((r->access & ~mask) == 0);
3890     }
3891
3892     /* Check that the register definition has enough info to handle
3893      * reads and writes if they are permitted.
3894      */
3895     if (!(r->type & (ARM_CP_SPECIAL|ARM_CP_CONST))) {
3896         if (r->access & PL3_R) {
3897             assert((r->fieldoffset ||
3898                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
3899                    r->readfn);
3900         }
3901         if (r->access & PL3_W) {
3902             assert((r->fieldoffset ||
3903                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
3904                    r->writefn);
3905         }
3906     }
3907     /* Bad type field probably means missing sentinel at end of reg list */
3908     assert(cptype_valid(r->type));
3909     for (crm = crmmin; crm <= crmmax; crm++) {
3910         for (opc1 = opc1min; opc1 <= opc1max; opc1++) {
3911             for (opc2 = opc2min; opc2 <= opc2max; opc2++) {
3912                 for (state = ARM_CP_STATE_AA32;
3913                      state <= ARM_CP_STATE_AA64; state++) {
3914                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
3915                         continue;
3916                     }
3917                     if (state == ARM_CP_STATE_AA32) {
3918                         /* Under AArch32 CP registers can be common
3919                          * (same for secure and non-secure world) or banked.
3920                          */
3921                         switch (r->secure) {
3922                         case ARM_CP_SECSTATE_S:
3923                         case ARM_CP_SECSTATE_NS:
3924                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3925                                                    r->secure, crm, opc1, opc2);
3926                             break;
3927                         default:
3928                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3929                                                    ARM_CP_SECSTATE_S,
3930                                                    crm, opc1, opc2);
3931                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3932                                                    ARM_CP_SECSTATE_NS,
3933                                                    crm, opc1, opc2);
3934                             break;
3935                         }
3936                     } else {
3937                         /* AArch64 registers get mapped to non-secure instance
3938                          * of AArch32 */
3939                         add_cpreg_to_hashtable(cpu, r, opaque, state,
3940                                                ARM_CP_SECSTATE_NS,
3941                                                crm, opc1, opc2);
3942                     }
3943                 }
3944             }
3945         }
3946     }
3947 }
3948
3949 void define_arm_cp_regs_with_opaque(ARMCPU *cpu,
3950                                     const ARMCPRegInfo *regs, void *opaque)
3951 {
3952     /* Define a whole list of registers */
3953     const ARMCPRegInfo *r;
3954     for (r = regs; r->type != ARM_CP_SENTINEL; r++) {
3955         define_one_arm_cp_reg_with_opaque(cpu, r, opaque);
3956     }
3957 }
3958
3959 const ARMCPRegInfo *get_arm_cp_reginfo(GHashTable *cpregs, uint32_t encoded_cp)
3960 {
3961     return g_hash_table_lookup(cpregs, &encoded_cp);
3962 }
3963
3964 void arm_cp_write_ignore(CPUARMState *env, const ARMCPRegInfo *ri,
3965                          uint64_t value)
3966 {
3967     /* Helper coprocessor write function for write-ignore registers */
3968 }
3969
3970 uint64_t arm_cp_read_zero(CPUARMState *env, const ARMCPRegInfo *ri)
3971 {
3972     /* Helper coprocessor write function for read-as-zero registers */
3973     return 0;
3974 }
3975
3976 void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
3977 {
3978     /* Helper coprocessor reset function for do-nothing-on-reset registers */
3979 }
3980
3981 static int bad_mode_switch(CPUARMState *env, int mode)
3982 {
3983     /* Return true if it is not valid for us to switch to
3984      * this CPU mode (ie all the UNPREDICTABLE cases in
3985      * the ARM ARM CPSRWriteByInstr pseudocode).
3986      */
3987     switch (mode) {
3988     case ARM_CPU_MODE_USR:
3989     case ARM_CPU_MODE_SYS:
3990     case ARM_CPU_MODE_SVC:
3991     case ARM_CPU_MODE_ABT:
3992     case ARM_CPU_MODE_UND:
3993     case ARM_CPU_MODE_IRQ:
3994     case ARM_CPU_MODE_FIQ:
3995         return 0;
3996     case ARM_CPU_MODE_MON:
3997         return !arm_is_secure(env);
3998     default:
3999         return 1;
4000     }
4001 }
4002
4003 uint32_t cpsr_read(CPUARMState *env)
4004 {
4005     int ZF;
4006     ZF = (env->ZF == 0);
4007     return env->uncached_cpsr | (env->NF & 0x80000000) | (ZF << 30) |
4008         (env->CF << 29) | ((env->VF & 0x80000000) >> 3) | (env->QF << 27)
4009         | (env->thumb << 5) | ((env->condexec_bits & 3) << 25)
4010         | ((env->condexec_bits & 0xfc) << 8)
4011         | (env->GE << 16) | (env->daif & CPSR_AIF);
4012 }
4013
4014 void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
4015 {
4016     uint32_t changed_daif;
4017
4018     if (mask & CPSR_NZCV) {
4019         env->ZF = (~val) & CPSR_Z;
4020         env->NF = val;
4021         env->CF = (val >> 29) & 1;
4022         env->VF = (val << 3) & 0x80000000;
4023     }
4024     if (mask & CPSR_Q)
4025         env->QF = ((val & CPSR_Q) != 0);
4026     if (mask & CPSR_T)
4027         env->thumb = ((val & CPSR_T) != 0);
4028     if (mask & CPSR_IT_0_1) {
4029         env->condexec_bits &= ~3;
4030         env->condexec_bits |= (val >> 25) & 3;
4031     }
4032     if (mask & CPSR_IT_2_7) {
4033         env->condexec_bits &= 3;
4034         env->condexec_bits |= (val >> 8) & 0xfc;
4035     }
4036     if (mask & CPSR_GE) {
4037         env->GE = (val >> 16) & 0xf;
4038     }
4039
4040     /* In a V7 implementation that includes the security extensions but does
4041      * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
4042      * whether non-secure software is allowed to change the CPSR_F and CPSR_A
4043      * bits respectively.
4044      *
4045      * In a V8 implementation, it is permitted for privileged software to
4046      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
4047      */
4048     if (!arm_feature(env, ARM_FEATURE_V8) &&
4049         arm_feature(env, ARM_FEATURE_EL3) &&
4050         !arm_feature(env, ARM_FEATURE_EL2) &&
4051         !arm_is_secure(env)) {
4052
4053         changed_daif = (env->daif ^ val) & mask;
4054
4055         if (changed_daif & CPSR_A) {
4056             /* Check to see if we are allowed to change the masking of async
4057              * abort exceptions from a non-secure state.
4058              */
4059             if (!(env->cp15.scr_el3 & SCR_AW)) {
4060                 qemu_log_mask(LOG_GUEST_ERROR,
4061                               "Ignoring attempt to switch CPSR_A flag from "
4062                               "non-secure world with SCR.AW bit clear\n");
4063                 mask &= ~CPSR_A;
4064             }
4065         }
4066
4067         if (changed_daif & CPSR_F) {
4068             /* Check to see if we are allowed to change the masking of FIQ
4069              * exceptions from a non-secure state.
4070              */
4071             if (!(env->cp15.scr_el3 & SCR_FW)) {
4072                 qemu_log_mask(LOG_GUEST_ERROR,
4073                               "Ignoring attempt to switch CPSR_F flag from "
4074                               "non-secure world with SCR.FW bit clear\n");
4075                 mask &= ~CPSR_F;
4076             }
4077
4078             /* Check whether non-maskable FIQ (NMFI) support is enabled.
4079              * If this bit is set software is not allowed to mask
4080              * FIQs, but is allowed to set CPSR_F to 0.
4081              */
4082             if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
4083                 (val & CPSR_F)) {
4084                 qemu_log_mask(LOG_GUEST_ERROR,
4085                               "Ignoring attempt to enable CPSR_F flag "
4086                               "(non-maskable FIQ [NMFI] support enabled)\n");
4087                 mask &= ~CPSR_F;
4088             }
4089         }
4090     }
4091
4092     env->daif &= ~(CPSR_AIF & mask);
4093     env->daif |= val & CPSR_AIF & mask;
4094
4095     if ((env->uncached_cpsr ^ val) & mask & CPSR_M) {
4096         if (bad_mode_switch(env, val & CPSR_M)) {
4097             /* Attempt to switch to an invalid mode: this is UNPREDICTABLE.
4098              * We choose to ignore the attempt and leave the CPSR M field
4099              * untouched.
4100              */
4101             mask &= ~CPSR_M;
4102         } else {
4103             switch_mode(env, val & CPSR_M);
4104         }
4105     }
4106     mask &= ~CACHED_CPSR_BITS;
4107     env->uncached_cpsr = (env->uncached_cpsr & ~mask) | (val & mask);
4108 }
4109
4110 /* Sign/zero extend */
4111 uint32_t HELPER(sxtb16)(uint32_t x)
4112 {
4113     uint32_t res;
4114     res = (uint16_t)(int8_t)x;
4115     res |= (uint32_t)(int8_t)(x >> 16) << 16;
4116     return res;
4117 }
4118
4119 uint32_t HELPER(uxtb16)(uint32_t x)
4120 {
4121     uint32_t res;
4122     res = (uint16_t)(uint8_t)x;
4123     res |= (uint32_t)(uint8_t)(x >> 16) << 16;
4124     return res;
4125 }
4126
4127 uint32_t HELPER(clz)(uint32_t x)
4128 {
4129     return clz32(x);
4130 }
4131
4132 int32_t HELPER(sdiv)(int32_t num, int32_t den)
4133 {
4134     if (den == 0)
4135       return 0;
4136     if (num == INT_MIN && den == -1)
4137       return INT_MIN;
4138     return num / den;
4139 }
4140
4141 uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
4142 {
4143     if (den == 0)
4144       return 0;
4145     return num / den;
4146 }
4147
4148 uint32_t HELPER(rbit)(uint32_t x)
4149 {
4150     x =  ((x & 0xff000000) >> 24)
4151        | ((x & 0x00ff0000) >> 8)
4152        | ((x & 0x0000ff00) << 8)
4153        | ((x & 0x000000ff) << 24);
4154     x =  ((x & 0xf0f0f0f0) >> 4)
4155        | ((x & 0x0f0f0f0f) << 4);
4156     x =  ((x & 0x88888888) >> 3)
4157        | ((x & 0x44444444) >> 1)
4158        | ((x & 0x22222222) << 1)
4159        | ((x & 0x11111111) << 3);
4160     return x;
4161 }
4162
4163 #if defined(CONFIG_USER_ONLY)
4164
4165 /* These should probably raise undefined insn exceptions.  */
4166 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
4167 {
4168     ARMCPU *cpu = arm_env_get_cpu(env);
4169
4170     cpu_abort(CPU(cpu), "v7m_msr %d\n", reg);
4171 }
4172
4173 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
4174 {
4175     ARMCPU *cpu = arm_env_get_cpu(env);
4176
4177     cpu_abort(CPU(cpu), "v7m_mrs %d\n", reg);
4178     return 0;
4179 }
4180
4181 void switch_mode(CPUARMState *env, int mode)
4182 {
4183     ARMCPU *cpu = arm_env_get_cpu(env);
4184
4185     if (mode != ARM_CPU_MODE_USR) {
4186         cpu_abort(CPU(cpu), "Tried to switch out of user mode\n");
4187     }
4188 }
4189
4190 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
4191 {
4192     ARMCPU *cpu = arm_env_get_cpu(env);
4193
4194     cpu_abort(CPU(cpu), "banked r13 write\n");
4195 }
4196
4197 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
4198 {
4199     ARMCPU *cpu = arm_env_get_cpu(env);
4200
4201     cpu_abort(CPU(cpu), "banked r13 read\n");
4202     return 0;
4203 }
4204
4205 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
4206                                  uint32_t cur_el, bool secure)
4207 {
4208     return 1;
4209 }
4210
4211 void aarch64_sync_64_to_32(CPUARMState *env)
4212 {
4213     g_assert_not_reached();
4214 }
4215
4216 #else
4217
4218 /* Map CPU modes onto saved register banks.  */
4219 int bank_number(int mode)
4220 {
4221     switch (mode) {
4222     case ARM_CPU_MODE_USR:
4223     case ARM_CPU_MODE_SYS:
4224         return 0;
4225     case ARM_CPU_MODE_SVC:
4226         return 1;
4227     case ARM_CPU_MODE_ABT:
4228         return 2;
4229     case ARM_CPU_MODE_UND:
4230         return 3;
4231     case ARM_CPU_MODE_IRQ:
4232         return 4;
4233     case ARM_CPU_MODE_FIQ:
4234         return 5;
4235     case ARM_CPU_MODE_HYP:
4236         return 6;
4237     case ARM_CPU_MODE_MON:
4238         return 7;
4239     }
4240     hw_error("bank number requested for bad CPSR mode value 0x%x\n", mode);
4241 }
4242
4243 void switch_mode(CPUARMState *env, int mode)
4244 {
4245     int old_mode;
4246     int i;
4247
4248     old_mode = env->uncached_cpsr & CPSR_M;
4249     if (mode == old_mode)
4250         return;
4251
4252     if (old_mode == ARM_CPU_MODE_FIQ) {
4253         memcpy (env->fiq_regs, env->regs + 8, 5 * sizeof(uint32_t));
4254         memcpy (env->regs + 8, env->usr_regs, 5 * sizeof(uint32_t));
4255     } else if (mode == ARM_CPU_MODE_FIQ) {
4256         memcpy (env->usr_regs, env->regs + 8, 5 * sizeof(uint32_t));
4257         memcpy (env->regs + 8, env->fiq_regs, 5 * sizeof(uint32_t));
4258     }
4259
4260     i = bank_number(old_mode);
4261     env->banked_r13[i] = env->regs[13];
4262     env->banked_r14[i] = env->regs[14];
4263     env->banked_spsr[i] = env->spsr;
4264
4265     i = bank_number(mode);
4266     env->regs[13] = env->banked_r13[i];
4267     env->regs[14] = env->banked_r14[i];
4268     env->spsr = env->banked_spsr[i];
4269 }
4270
4271 /* Physical Interrupt Target EL Lookup Table
4272  *
4273  * [ From ARM ARM section G1.13.4 (Table G1-15) ]
4274  *
4275  * The below multi-dimensional table is used for looking up the target
4276  * exception level given numerous condition criteria.  Specifically, the
4277  * target EL is based on SCR and HCR routing controls as well as the
4278  * currently executing EL and secure state.
4279  *
4280  *    Dimensions:
4281  *    target_el_table[2][2][2][2][2][4]
4282  *                    |  |  |  |  |  +--- Current EL
4283  *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
4284  *                    |  |  |  +--------- HCR mask override
4285  *                    |  |  +------------ SCR exec state control
4286  *                    |  +--------------- SCR mask override
4287  *                    +------------------ 32-bit(0)/64-bit(1) EL3
4288  *
4289  *    The table values are as such:
4290  *    0-3 = EL0-EL3
4291  *     -1 = Cannot occur
4292  *
4293  * The ARM ARM target EL table includes entries indicating that an "exception
4294  * is not taken".  The two cases where this is applicable are:
4295  *    1) An exception is taken from EL3 but the SCR does not have the exception
4296  *    routed to EL3.
4297  *    2) An exception is taken from EL2 but the HCR does not have the exception
4298  *    routed to EL2.
4299  * In these two cases, the below table contain a target of EL1.  This value is
4300  * returned as it is expected that the consumer of the table data will check
4301  * for "target EL >= current EL" to ensure the exception is not taken.
4302  *
4303  *            SCR     HCR
4304  *         64  EA     AMO                 From
4305  *        BIT IRQ     IMO      Non-secure         Secure
4306  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
4307  */
4308 const int8_t target_el_table[2][2][2][2][2][4] = {
4309     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
4310        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
4311       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
4312        {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
4313      {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
4314        {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
4315       {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
4316        {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
4317     {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
4318        {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},
4319       {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1, -1,  1 },},
4320        {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},},
4321      {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
4322        {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
4323       {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
4324        {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},},},
4325 };
4326
4327 /*
4328  * Determine the target EL for physical exceptions
4329  */
4330 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
4331                                  uint32_t cur_el, bool secure)
4332 {
4333     CPUARMState *env = cs->env_ptr;
4334     int rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
4335     int scr;
4336     int hcr;
4337     int target_el;
4338     int is64 = arm_el_is_aa64(env, 3);
4339
4340     switch (excp_idx) {
4341     case EXCP_IRQ:
4342         scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
4343         hcr = ((env->cp15.hcr_el2 & HCR_IMO) == HCR_IMO);
4344         break;
4345     case EXCP_FIQ:
4346         scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
4347         hcr = ((env->cp15.hcr_el2 & HCR_FMO) == HCR_FMO);
4348         break;
4349     default:
4350         scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
4351         hcr = ((env->cp15.hcr_el2 & HCR_AMO) == HCR_AMO);
4352         break;
4353     };
4354
4355     /* If HCR.TGE is set then HCR is treated as being 1 */
4356     hcr |= ((env->cp15.hcr_el2 & HCR_TGE) == HCR_TGE);
4357
4358     /* Perform a table-lookup for the target EL given the current state */
4359     target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
4360
4361     assert(target_el > 0);
4362
4363     return target_el;
4364 }
4365
4366 static void v7m_push(CPUARMState *env, uint32_t val)
4367 {
4368     CPUState *cs = CPU(arm_env_get_cpu(env));
4369
4370     env->regs[13] -= 4;
4371     stl_phys(cs->as, env->regs[13], val);
4372 }
4373
4374 static uint32_t v7m_pop(CPUARMState *env)
4375 {
4376     CPUState *cs = CPU(arm_env_get_cpu(env));
4377     uint32_t val;
4378
4379     val = ldl_phys(cs->as, env->regs[13]);
4380     env->regs[13] += 4;
4381     return val;
4382 }
4383
4384 /* Switch to V7M main or process stack pointer.  */
4385 static void switch_v7m_sp(CPUARMState *env, int process)
4386 {
4387     uint32_t tmp;
4388     if (env->v7m.current_sp != process) {
4389         tmp = env->v7m.other_sp;
4390         env->v7m.other_sp = env->regs[13];
4391         env->regs[13] = tmp;
4392         env->v7m.current_sp = process;
4393     }
4394 }
4395
4396 static void do_v7m_exception_exit(CPUARMState *env)
4397 {
4398     uint32_t type;
4399     uint32_t xpsr;
4400
4401     type = env->regs[15];
4402     if (env->v7m.exception != 0)
4403         armv7m_nvic_complete_irq(env->nvic, env->v7m.exception);
4404
4405     /* Switch to the target stack.  */
4406     switch_v7m_sp(env, (type & 4) != 0);
4407     /* Pop registers.  */
4408     env->regs[0] = v7m_pop(env);
4409     env->regs[1] = v7m_pop(env);
4410     env->regs[2] = v7m_pop(env);
4411     env->regs[3] = v7m_pop(env);
4412     env->regs[12] = v7m_pop(env);
4413     env->regs[14] = v7m_pop(env);
4414     env->regs[15] = v7m_pop(env);
4415     if (env->regs[15] & 1) {
4416         qemu_log_mask(LOG_GUEST_ERROR,
4417                       "M profile return from interrupt with misaligned "
4418                       "PC is UNPREDICTABLE\n");
4419         /* Actual hardware seems to ignore the lsbit, and there are several
4420          * RTOSes out there which incorrectly assume the r15 in the stack
4421          * frame should be a Thumb-style "lsbit indicates ARM/Thumb" value.
4422          */
4423         env->regs[15] &= ~1U;
4424     }
4425     xpsr = v7m_pop(env);
4426     xpsr_write(env, xpsr, 0xfffffdff);
4427     /* Undo stack alignment.  */
4428     if (xpsr & 0x200)
4429         env->regs[13] |= 4;
4430     /* ??? The exception return type specifies Thread/Handler mode.  However
4431        this is also implied by the xPSR value. Not sure what to do
4432        if there is a mismatch.  */
4433     /* ??? Likewise for mismatches between the CONTROL register and the stack
4434        pointer.  */
4435 }
4436
4437 void arm_v7m_cpu_do_interrupt(CPUState *cs)
4438 {
4439     ARMCPU *cpu = ARM_CPU(cs);
4440     CPUARMState *env = &cpu->env;
4441     uint32_t xpsr = xpsr_read(env);
4442     uint32_t lr;
4443     uint32_t addr;
4444
4445     arm_log_exception(cs->exception_index);
4446
4447     lr = 0xfffffff1;
4448     if (env->v7m.current_sp)
4449         lr |= 4;
4450     if (env->v7m.exception == 0)
4451         lr |= 8;
4452
4453     /* For exceptions we just mark as pending on the NVIC, and let that
4454        handle it.  */
4455     /* TODO: Need to escalate if the current priority is higher than the
4456        one we're raising.  */
4457     switch (cs->exception_index) {
4458     case EXCP_UDEF:
4459         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE);
4460         return;
4461     case EXCP_SWI:
4462         /* The PC already points to the next instruction.  */
4463         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SVC);
4464         return;
4465     case EXCP_PREFETCH_ABORT:
4466     case EXCP_DATA_ABORT:
4467         /* TODO: if we implemented the MPU registers, this is where we
4468          * should set the MMFAR, etc from exception.fsr and exception.vaddress.
4469          */
4470         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_MEM);
4471         return;
4472     case EXCP_BKPT:
4473         if (semihosting_enabled) {
4474             int nr;
4475             nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
4476             if (nr == 0xab) {
4477                 env->regs[15] += 2;
4478                 env->regs[0] = do_arm_semihosting(env);
4479                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4480                 return;
4481             }
4482         }
4483         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_DEBUG);
4484         return;
4485     case EXCP_IRQ:
4486         env->v7m.exception = armv7m_nvic_acknowledge_irq(env->nvic);
4487         break;
4488     case EXCP_EXCEPTION_EXIT:
4489         do_v7m_exception_exit(env);
4490         return;
4491     default:
4492         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
4493         return; /* Never happens.  Keep compiler happy.  */
4494     }
4495
4496     /* Align stack pointer.  */
4497     /* ??? Should only do this if Configuration Control Register
4498        STACKALIGN bit is set.  */
4499     if (env->regs[13] & 4) {
4500         env->regs[13] -= 4;
4501         xpsr |= 0x200;
4502     }
4503     /* Switch to the handler mode.  */
4504     v7m_push(env, xpsr);
4505     v7m_push(env, env->regs[15]);
4506     v7m_push(env, env->regs[14]);
4507     v7m_push(env, env->regs[12]);
4508     v7m_push(env, env->regs[3]);
4509     v7m_push(env, env->regs[2]);
4510     v7m_push(env, env->regs[1]);
4511     v7m_push(env, env->regs[0]);
4512     switch_v7m_sp(env, 0);
4513     /* Clear IT bits */
4514     env->condexec_bits = 0;
4515     env->regs[14] = lr;
4516     addr = ldl_phys(cs->as, env->v7m.vecbase + env->v7m.exception * 4);
4517     env->regs[15] = addr & 0xfffffffe;
4518     env->thumb = addr & 1;
4519 }
4520
4521 /* Function used to synchronize QEMU's AArch64 register set with AArch32
4522  * register set.  This is necessary when switching between AArch32 and AArch64
4523  * execution state.
4524  */
4525 void aarch64_sync_32_to_64(CPUARMState *env)
4526 {
4527     int i;
4528     uint32_t mode = env->uncached_cpsr & CPSR_M;
4529
4530     /* We can blanket copy R[0:7] to X[0:7] */
4531     for (i = 0; i < 8; i++) {
4532         env->xregs[i] = env->regs[i];
4533     }
4534
4535     /* Unless we are in FIQ mode, x8-x12 come from the user registers r8-r12.
4536      * Otherwise, they come from the banked user regs.
4537      */
4538     if (mode == ARM_CPU_MODE_FIQ) {
4539         for (i = 8; i < 13; i++) {
4540             env->xregs[i] = env->usr_regs[i - 8];
4541         }
4542     } else {
4543         for (i = 8; i < 13; i++) {
4544             env->xregs[i] = env->regs[i];
4545         }
4546     }
4547
4548     /* Registers x13-x23 are the various mode SP and FP registers. Registers
4549      * r13 and r14 are only copied if we are in that mode, otherwise we copy
4550      * from the mode banked register.
4551      */
4552     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
4553         env->xregs[13] = env->regs[13];
4554         env->xregs[14] = env->regs[14];
4555     } else {
4556         env->xregs[13] = env->banked_r13[bank_number(ARM_CPU_MODE_USR)];
4557         /* HYP is an exception in that it is copied from r14 */
4558         if (mode == ARM_CPU_MODE_HYP) {
4559             env->xregs[14] = env->regs[14];
4560         } else {
4561             env->xregs[14] = env->banked_r14[bank_number(ARM_CPU_MODE_USR)];
4562         }
4563     }
4564
4565     if (mode == ARM_CPU_MODE_HYP) {
4566         env->xregs[15] = env->regs[13];
4567     } else {
4568         env->xregs[15] = env->banked_r13[bank_number(ARM_CPU_MODE_HYP)];
4569     }
4570
4571     if (mode == ARM_CPU_MODE_IRQ) {
4572         env->xregs[16] = env->regs[13];
4573         env->xregs[17] = env->regs[14];
4574     } else {
4575         env->xregs[16] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
4576         env->xregs[17] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
4577     }
4578
4579     if (mode == ARM_CPU_MODE_SVC) {
4580         env->xregs[18] = env->regs[13];
4581         env->xregs[19] = env->regs[14];
4582     } else {
4583         env->xregs[18] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
4584         env->xregs[19] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
4585     }
4586
4587     if (mode == ARM_CPU_MODE_ABT) {
4588         env->xregs[20] = env->regs[13];
4589         env->xregs[21] = env->regs[14];
4590     } else {
4591         env->xregs[20] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
4592         env->xregs[21] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
4593     }
4594
4595     if (mode == ARM_CPU_MODE_UND) {
4596         env->xregs[22] = env->regs[13];
4597         env->xregs[23] = env->regs[14];
4598     } else {
4599         env->xregs[22] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
4600         env->xregs[23] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
4601     }
4602
4603     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
4604      * mode, then we can copy from r8-r14.  Otherwise, we copy from the
4605      * FIQ bank for r8-r14.
4606      */
4607     if (mode == ARM_CPU_MODE_FIQ) {
4608         for (i = 24; i < 31; i++) {
4609             env->xregs[i] = env->regs[i - 16];   /* X[24:30] <- R[8:14] */
4610         }
4611     } else {
4612         for (i = 24; i < 29; i++) {
4613             env->xregs[i] = env->fiq_regs[i - 24];
4614         }
4615         env->xregs[29] = env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)];
4616         env->xregs[30] = env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)];
4617     }
4618
4619     env->pc = env->regs[15];
4620 }
4621
4622 /* Function used to synchronize QEMU's AArch32 register set with AArch64
4623  * register set.  This is necessary when switching between AArch32 and AArch64
4624  * execution state.
4625  */
4626 void aarch64_sync_64_to_32(CPUARMState *env)
4627 {
4628     int i;
4629     uint32_t mode = env->uncached_cpsr & CPSR_M;
4630
4631     /* We can blanket copy X[0:7] to R[0:7] */
4632     for (i = 0; i < 8; i++) {
4633         env->regs[i] = env->xregs[i];
4634     }
4635
4636     /* Unless we are in FIQ mode, r8-r12 come from the user registers x8-x12.
4637      * Otherwise, we copy x8-x12 into the banked user regs.
4638      */
4639     if (mode == ARM_CPU_MODE_FIQ) {
4640         for (i = 8; i < 13; i++) {
4641             env->usr_regs[i - 8] = env->xregs[i];
4642         }
4643     } else {
4644         for (i = 8; i < 13; i++) {
4645             env->regs[i] = env->xregs[i];
4646         }
4647     }
4648
4649     /* Registers r13 & r14 depend on the current mode.
4650      * If we are in a given mode, we copy the corresponding x registers to r13
4651      * and r14.  Otherwise, we copy the x register to the banked r13 and r14
4652      * for the mode.
4653      */
4654     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
4655         env->regs[13] = env->xregs[13];
4656         env->regs[14] = env->xregs[14];
4657     } else {
4658         env->banked_r13[bank_number(ARM_CPU_MODE_USR)] = env->xregs[13];
4659
4660         /* HYP is an exception in that it does not have its own banked r14 but
4661          * shares the USR r14
4662          */
4663         if (mode == ARM_CPU_MODE_HYP) {
4664             env->regs[14] = env->xregs[14];
4665         } else {
4666             env->banked_r14[bank_number(ARM_CPU_MODE_USR)] = env->xregs[14];
4667         }
4668     }
4669
4670     if (mode == ARM_CPU_MODE_HYP) {
4671         env->regs[13] = env->xregs[15];
4672     } else {
4673         env->banked_r13[bank_number(ARM_CPU_MODE_HYP)] = env->xregs[15];
4674     }
4675
4676     if (mode == ARM_CPU_MODE_IRQ) {
4677         env->regs[13] = env->xregs[16];
4678         env->regs[14] = env->xregs[17];
4679     } else {
4680         env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
4681         env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
4682     }
4683
4684     if (mode == ARM_CPU_MODE_SVC) {
4685         env->regs[13] = env->xregs[18];
4686         env->regs[14] = env->xregs[19];
4687     } else {
4688         env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
4689         env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
4690     }
4691
4692     if (mode == ARM_CPU_MODE_ABT) {
4693         env->regs[13] = env->xregs[20];
4694         env->regs[14] = env->xregs[21];
4695     } else {
4696         env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
4697         env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
4698     }
4699
4700     if (mode == ARM_CPU_MODE_UND) {
4701         env->regs[13] = env->xregs[22];
4702         env->regs[14] = env->xregs[23];
4703     } else {
4704         env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
4705         env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
4706     }
4707
4708     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
4709      * mode, then we can copy to r8-r14.  Otherwise, we copy to the
4710      * FIQ bank for r8-r14.
4711      */
4712     if (mode == ARM_CPU_MODE_FIQ) {
4713         for (i = 24; i < 31; i++) {
4714             env->regs[i - 16] = env->xregs[i];   /* X[24:30] -> R[8:14] */
4715         }
4716     } else {
4717         for (i = 24; i < 29; i++) {
4718             env->fiq_regs[i - 24] = env->xregs[i];
4719         }
4720         env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[29];
4721         env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[30];
4722     }
4723
4724     env->regs[15] = env->pc;
4725 }
4726
4727 /* Handle a CPU exception.  */
4728 void arm_cpu_do_interrupt(CPUState *cs)
4729 {
4730     ARMCPU *cpu = ARM_CPU(cs);
4731     CPUARMState *env = &cpu->env;
4732     uint32_t addr;
4733     uint32_t mask;
4734     int new_mode;
4735     uint32_t offset;
4736     uint32_t moe;
4737
4738     assert(!IS_M(env));
4739
4740     arm_log_exception(cs->exception_index);
4741
4742     if (arm_is_psci_call(cpu, cs->exception_index)) {
4743         arm_handle_psci_call(cpu);
4744         qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
4745         return;
4746     }
4747
4748     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
4749     switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
4750     case EC_BREAKPOINT:
4751     case EC_BREAKPOINT_SAME_EL:
4752         moe = 1;
4753         break;
4754     case EC_WATCHPOINT:
4755     case EC_WATCHPOINT_SAME_EL:
4756         moe = 10;
4757         break;
4758     case EC_AA32_BKPT:
4759         moe = 3;
4760         break;
4761     case EC_VECTORCATCH:
4762         moe = 5;
4763         break;
4764     default:
4765         moe = 0;
4766         break;
4767     }
4768
4769     if (moe) {
4770         env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
4771     }
4772
4773     /* TODO: Vectored interrupt controller.  */
4774     switch (cs->exception_index) {
4775     case EXCP_UDEF:
4776         new_mode = ARM_CPU_MODE_UND;
4777         addr = 0x04;
4778         mask = CPSR_I;
4779         if (env->thumb)
4780             offset = 2;
4781         else
4782             offset = 4;
4783         break;
4784     case EXCP_SWI:
4785         if (semihosting_enabled) {
4786             /* Check for semihosting interrupt.  */
4787             if (env->thumb) {
4788                 mask = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
4789                     & 0xff;
4790             } else {
4791                 mask = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
4792                     & 0xffffff;
4793             }
4794             /* Only intercept calls from privileged modes, to provide some
4795                semblance of security.  */
4796             if (((mask == 0x123456 && !env->thumb)
4797                     || (mask == 0xab && env->thumb))
4798                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
4799                 env->regs[0] = do_arm_semihosting(env);
4800                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4801                 return;
4802             }
4803         }
4804         new_mode = ARM_CPU_MODE_SVC;
4805         addr = 0x08;
4806         mask = CPSR_I;
4807         /* The PC already points to the next instruction.  */
4808         offset = 0;
4809         break;
4810     case EXCP_BKPT:
4811         /* See if this is a semihosting syscall.  */
4812         if (env->thumb && semihosting_enabled) {
4813             mask = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
4814             if (mask == 0xab
4815                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
4816                 env->regs[15] += 2;
4817                 env->regs[0] = do_arm_semihosting(env);
4818                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4819                 return;
4820             }
4821         }
4822         env->exception.fsr = 2;
4823         /* Fall through to prefetch abort.  */
4824     case EXCP_PREFETCH_ABORT:
4825         A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
4826         A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
4827         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
4828                       env->exception.fsr, (uint32_t)env->exception.vaddress);
4829         new_mode = ARM_CPU_MODE_ABT;
4830         addr = 0x0c;
4831         mask = CPSR_A | CPSR_I;
4832         offset = 4;
4833         break;
4834     case EXCP_DATA_ABORT:
4835         A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
4836         A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
4837         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
4838                       env->exception.fsr,
4839                       (uint32_t)env->exception.vaddress);
4840         new_mode = ARM_CPU_MODE_ABT;
4841         addr = 0x10;
4842         mask = CPSR_A | CPSR_I;
4843         offset = 8;
4844         break;
4845     case EXCP_IRQ:
4846         new_mode = ARM_CPU_MODE_IRQ;
4847         addr = 0x18;
4848         /* Disable IRQ and imprecise data aborts.  */
4849         mask = CPSR_A | CPSR_I;
4850         offset = 4;
4851         if (env->cp15.scr_el3 & SCR_IRQ) {
4852             /* IRQ routed to monitor mode */
4853             new_mode = ARM_CPU_MODE_MON;
4854             mask |= CPSR_F;
4855         }
4856         break;
4857     case EXCP_FIQ:
4858         new_mode = ARM_CPU_MODE_FIQ;
4859         addr = 0x1c;
4860         /* Disable FIQ, IRQ and imprecise data aborts.  */
4861         mask = CPSR_A | CPSR_I | CPSR_F;
4862         if (env->cp15.scr_el3 & SCR_FIQ) {
4863             /* FIQ routed to monitor mode */
4864             new_mode = ARM_CPU_MODE_MON;
4865         }
4866         offset = 4;
4867         break;
4868     case EXCP_SMC:
4869         new_mode = ARM_CPU_MODE_MON;
4870         addr = 0x08;
4871         mask = CPSR_A | CPSR_I | CPSR_F;
4872         offset = 0;
4873         break;
4874     default:
4875         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
4876         return; /* Never happens.  Keep compiler happy.  */
4877     }
4878
4879     if (new_mode == ARM_CPU_MODE_MON) {
4880         addr += env->cp15.mvbar;
4881     } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
4882         /* High vectors. When enabled, base address cannot be remapped. */
4883         addr += 0xffff0000;
4884     } else {
4885         /* ARM v7 architectures provide a vector base address register to remap
4886          * the interrupt vector table.
4887          * This register is only followed in non-monitor mode, and is banked.
4888          * Note: only bits 31:5 are valid.
4889          */
4890         addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
4891     }
4892
4893     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
4894         env->cp15.scr_el3 &= ~SCR_NS;
4895     }
4896
4897     switch_mode (env, new_mode);
4898     /* For exceptions taken to AArch32 we must clear the SS bit in both
4899      * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
4900      */
4901     env->uncached_cpsr &= ~PSTATE_SS;
4902     env->spsr = cpsr_read(env);
4903     /* Clear IT bits.  */
4904     env->condexec_bits = 0;
4905     /* Switch to the new mode, and to the correct instruction set.  */
4906     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
4907     env->daif |= mask;
4908     /* this is a lie, as the was no c1_sys on V4T/V5, but who cares
4909      * and we should just guard the thumb mode on V4 */
4910     if (arm_feature(env, ARM_FEATURE_V4T)) {
4911         env->thumb = (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
4912     }
4913     env->regs[14] = env->regs[15] + offset;
4914     env->regs[15] = addr;
4915     cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
4916 }
4917
4918
4919 /* Return the exception level which controls this address translation regime */
4920 static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx)
4921 {
4922     switch (mmu_idx) {
4923     case ARMMMUIdx_S2NS:
4924     case ARMMMUIdx_S1E2:
4925         return 2;
4926     case ARMMMUIdx_S1E3:
4927         return 3;
4928     case ARMMMUIdx_S1SE0:
4929         return arm_el_is_aa64(env, 3) ? 1 : 3;
4930     case ARMMMUIdx_S1SE1:
4931     case ARMMMUIdx_S1NSE0:
4932     case ARMMMUIdx_S1NSE1:
4933         return 1;
4934     default:
4935         g_assert_not_reached();
4936     }
4937 }
4938
4939 /* Return true if this address translation regime is secure */
4940 static inline bool regime_is_secure(CPUARMState *env, ARMMMUIdx mmu_idx)
4941 {
4942     switch (mmu_idx) {
4943     case ARMMMUIdx_S12NSE0:
4944     case ARMMMUIdx_S12NSE1:
4945     case ARMMMUIdx_S1NSE0:
4946     case ARMMMUIdx_S1NSE1:
4947     case ARMMMUIdx_S1E2:
4948     case ARMMMUIdx_S2NS:
4949         return false;
4950     case ARMMMUIdx_S1E3:
4951     case ARMMMUIdx_S1SE0:
4952     case ARMMMUIdx_S1SE1:
4953         return true;
4954     default:
4955         g_assert_not_reached();
4956     }
4957 }
4958
4959 /* Return the SCTLR value which controls this address translation regime */
4960 static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx)
4961 {
4962     return env->cp15.sctlr_el[regime_el(env, mmu_idx)];
4963 }
4964
4965 /* Return true if the specified stage of address translation is disabled */
4966 static inline bool regime_translation_disabled(CPUARMState *env,
4967                                                ARMMMUIdx mmu_idx)
4968 {
4969     if (mmu_idx == ARMMMUIdx_S2NS) {
4970         return (env->cp15.hcr_el2 & HCR_VM) == 0;
4971     }
4972     return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
4973 }
4974
4975 /* Return the TCR controlling this translation regime */
4976 static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx)
4977 {
4978     if (mmu_idx == ARMMMUIdx_S2NS) {
4979         /* TODO: return VTCR_EL2 */
4980         g_assert_not_reached();
4981     }
4982     return &env->cp15.tcr_el[regime_el(env, mmu_idx)];
4983 }
4984
4985 /* Return the TTBR associated with this translation regime */
4986 static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx,
4987                                    int ttbrn)
4988 {
4989     if (mmu_idx == ARMMMUIdx_S2NS) {
4990         /* TODO: return VTTBR_EL2 */
4991         g_assert_not_reached();
4992     }
4993     if (ttbrn == 0) {
4994         return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
4995     } else {
4996         return env->cp15.ttbr1_el[regime_el(env, mmu_idx)];
4997     }
4998 }
4999
5000 /* Return true if the translation regime is using LPAE format page tables */
5001 static inline bool regime_using_lpae_format(CPUARMState *env,
5002                                             ARMMMUIdx mmu_idx)
5003 {
5004     int el = regime_el(env, mmu_idx);
5005     if (el == 2 || arm_el_is_aa64(env, el)) {
5006         return true;
5007     }
5008     if (arm_feature(env, ARM_FEATURE_LPAE)
5009         && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) {
5010         return true;
5011     }
5012     return false;
5013 }
5014
5015 static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx)
5016 {
5017     switch (mmu_idx) {
5018     case ARMMMUIdx_S1SE0:
5019     case ARMMMUIdx_S1NSE0:
5020         return true;
5021     default:
5022         return false;
5023     case ARMMMUIdx_S12NSE0:
5024     case ARMMMUIdx_S12NSE1:
5025         g_assert_not_reached();
5026     }
5027 }
5028
5029 /* Translate section/page access permissions to page
5030  * R/W protection flags
5031  *
5032  * @env:         CPUARMState
5033  * @mmu_idx:     MMU index indicating required translation regime
5034  * @ap:          The 3-bit access permissions (AP[2:0])
5035  * @domain_prot: The 2-bit domain access permissions
5036  */
5037 static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
5038                                 int ap, int domain_prot)
5039 {
5040     bool is_user = regime_is_user(env, mmu_idx);
5041
5042     if (domain_prot == 3) {
5043         return PAGE_READ | PAGE_WRITE;
5044     }
5045
5046     switch (ap) {
5047     case 0:
5048         if (arm_feature(env, ARM_FEATURE_V7)) {
5049             return 0;
5050         }
5051         switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) {
5052         case SCTLR_S:
5053             return is_user ? 0 : PAGE_READ;
5054         case SCTLR_R:
5055             return PAGE_READ;
5056         default:
5057             return 0;
5058         }
5059     case 1:
5060         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
5061     case 2:
5062         if (is_user) {
5063             return PAGE_READ;
5064         } else {
5065             return PAGE_READ | PAGE_WRITE;
5066         }
5067     case 3:
5068         return PAGE_READ | PAGE_WRITE;
5069     case 4: /* Reserved.  */
5070         return 0;
5071     case 5:
5072         return is_user ? 0 : PAGE_READ;
5073     case 6:
5074         return PAGE_READ;
5075     case 7:
5076         if (!arm_feature(env, ARM_FEATURE_V6K)) {
5077             return 0;
5078         }
5079         return PAGE_READ;
5080     default:
5081         g_assert_not_reached();
5082     }
5083 }
5084
5085 /* Translate section/page access permissions to page
5086  * R/W protection flags.
5087  *
5088  * @ap:      The 2-bit simple AP (AP[2:1])
5089  * @is_user: TRUE if accessing from PL0
5090  */
5091 static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
5092 {
5093     switch (ap) {
5094     case 0:
5095         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
5096     case 1:
5097         return PAGE_READ | PAGE_WRITE;
5098     case 2:
5099         return is_user ? 0 : PAGE_READ;
5100     case 3:
5101         return PAGE_READ;
5102     default:
5103         g_assert_not_reached();
5104     }
5105 }
5106
5107 static inline int
5108 simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
5109 {
5110     return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
5111 }
5112
5113 /* Translate section/page access permissions to protection flags
5114  *
5115  * @env:     CPUARMState
5116  * @mmu_idx: MMU index indicating required translation regime
5117  * @is_aa64: TRUE if AArch64
5118  * @ap:      The 2-bit simple AP (AP[2:1])
5119  * @ns:      NS (non-secure) bit
5120  * @xn:      XN (execute-never) bit
5121  * @pxn:     PXN (privileged execute-never) bit
5122  */
5123 static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
5124                       int ap, int ns, int xn, int pxn)
5125 {
5126     bool is_user = regime_is_user(env, mmu_idx);
5127     int prot_rw, user_rw;
5128     bool have_wxn;
5129     int wxn = 0;
5130
5131     assert(mmu_idx != ARMMMUIdx_S2NS);
5132
5133     user_rw = simple_ap_to_rw_prot_is_user(ap, true);
5134     if (is_user) {
5135         prot_rw = user_rw;
5136     } else {
5137         prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
5138     }
5139
5140     if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) {
5141         return prot_rw;
5142     }
5143
5144     /* TODO have_wxn should be replaced with
5145      *   ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
5146      * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
5147      * compatible processors have EL2, which is required for [U]WXN.
5148      */
5149     have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
5150
5151     if (have_wxn) {
5152         wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
5153     }
5154
5155     if (is_aa64) {
5156         switch (regime_el(env, mmu_idx)) {
5157         case 1:
5158             if (!is_user) {
5159                 xn = pxn || (user_rw & PAGE_WRITE);
5160             }
5161             break;
5162         case 2:
5163         case 3:
5164             break;
5165         }
5166     } else if (arm_feature(env, ARM_FEATURE_V7)) {
5167         switch (regime_el(env, mmu_idx)) {
5168         case 1:
5169         case 3:
5170             if (is_user) {
5171                 xn = xn || !(user_rw & PAGE_READ);
5172             } else {
5173                 int uwxn = 0;
5174                 if (have_wxn) {
5175                     uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
5176                 }
5177                 xn = xn || !(prot_rw & PAGE_READ) || pxn ||
5178                      (uwxn && (user_rw & PAGE_WRITE));
5179             }
5180             break;
5181         case 2:
5182             break;
5183         }
5184     } else {
5185         xn = wxn = 0;
5186     }
5187
5188     if (xn || (wxn && (prot_rw & PAGE_WRITE))) {
5189         return prot_rw;
5190     }
5191     return prot_rw | PAGE_EXEC;
5192 }
5193
5194 static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
5195                                      uint32_t *table, uint32_t address)
5196 {
5197     /* Note that we can only get here for an AArch32 PL0/PL1 lookup */
5198     TCR *tcr = regime_tcr(env, mmu_idx);
5199
5200     if (address & tcr->mask) {
5201         if (tcr->raw_tcr & TTBCR_PD1) {
5202             /* Translation table walk disabled for TTBR1 */
5203             return false;
5204         }
5205         *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000;
5206     } else {
5207         if (tcr->raw_tcr & TTBCR_PD0) {
5208             /* Translation table walk disabled for TTBR0 */
5209             return false;
5210         }
5211         *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask;
5212     }
5213     *table |= (address >> 18) & 0x3ffc;
5214     return true;
5215 }
5216
5217 /* All loads done in the course of a page table walk go through here.
5218  * TODO: rather than ignoring errors from physical memory reads (which
5219  * are external aborts in ARM terminology) we should propagate this
5220  * error out so that we can turn it into a Data Abort if this walk
5221  * was being done for a CPU load/store or an address translation instruction
5222  * (but not if it was for a debug access).
5223  */
5224 static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure)
5225 {
5226     MemTxAttrs attrs = {};
5227
5228     attrs.secure = is_secure;
5229     return address_space_ldl(cs->as, addr, attrs, NULL);
5230 }
5231
5232 static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure)
5233 {
5234     MemTxAttrs attrs = {};
5235
5236     attrs.secure = is_secure;
5237     return address_space_ldq(cs->as, addr, attrs, NULL);
5238 }
5239
5240 static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
5241                             ARMMMUIdx mmu_idx, hwaddr *phys_ptr,
5242                             int *prot, target_ulong *page_size)
5243 {
5244     CPUState *cs = CPU(arm_env_get_cpu(env));
5245     int code;
5246     uint32_t table;
5247     uint32_t desc;
5248     int type;
5249     int ap;
5250     int domain = 0;
5251     int domain_prot;
5252     hwaddr phys_addr;
5253     uint32_t dacr;
5254
5255     /* Pagetable walk.  */
5256     /* Lookup l1 descriptor.  */
5257     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
5258         /* Section translation fault if page walk is disabled by PD0 or PD1 */
5259         code = 5;
5260         goto do_fault;
5261     }
5262     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5263     type = (desc & 3);
5264     domain = (desc >> 5) & 0x0f;
5265     if (regime_el(env, mmu_idx) == 1) {
5266         dacr = env->cp15.dacr_ns;
5267     } else {
5268         dacr = env->cp15.dacr_s;
5269     }
5270     domain_prot = (dacr >> (domain * 2)) & 3;
5271     if (type == 0) {
5272         /* Section translation fault.  */
5273         code = 5;
5274         goto do_fault;
5275     }
5276     if (domain_prot == 0 || domain_prot == 2) {
5277         if (type == 2)
5278             code = 9; /* Section domain fault.  */
5279         else
5280             code = 11; /* Page domain fault.  */
5281         goto do_fault;
5282     }
5283     if (type == 2) {
5284         /* 1Mb section.  */
5285         phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
5286         ap = (desc >> 10) & 3;
5287         code = 13;
5288         *page_size = 1024 * 1024;
5289     } else {
5290         /* Lookup l2 entry.  */
5291         if (type == 1) {
5292             /* Coarse pagetable.  */
5293             table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
5294         } else {
5295             /* Fine pagetable.  */
5296             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
5297         }
5298         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5299         switch (desc & 3) {
5300         case 0: /* Page translation fault.  */
5301             code = 7;
5302             goto do_fault;
5303         case 1: /* 64k page.  */
5304             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
5305             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
5306             *page_size = 0x10000;
5307             break;
5308         case 2: /* 4k page.  */
5309             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
5310             ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
5311             *page_size = 0x1000;
5312             break;
5313         case 3: /* 1k page.  */
5314             if (type == 1) {
5315                 if (arm_feature(env, ARM_FEATURE_XSCALE)) {
5316                     phys_addr = (desc & 0xfffff000) | (address & 0xfff);
5317                 } else {
5318                     /* Page translation fault.  */
5319                     code = 7;
5320                     goto do_fault;
5321                 }
5322             } else {
5323                 phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
5324             }
5325             ap = (desc >> 4) & 3;
5326             *page_size = 0x400;
5327             break;
5328         default:
5329             /* Never happens, but compiler isn't smart enough to tell.  */
5330             abort();
5331         }
5332         code = 15;
5333     }
5334     *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
5335     *prot |= *prot ? PAGE_EXEC : 0;
5336     if (!(*prot & (1 << access_type))) {
5337         /* Access permission fault.  */
5338         goto do_fault;
5339     }
5340     *phys_ptr = phys_addr;
5341     return 0;
5342 do_fault:
5343     return code | (domain << 4);
5344 }
5345
5346 static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
5347                             ARMMMUIdx mmu_idx, hwaddr *phys_ptr,
5348                             MemTxAttrs *attrs,
5349                             int *prot, target_ulong *page_size)
5350 {
5351     CPUState *cs = CPU(arm_env_get_cpu(env));
5352     int code;
5353     uint32_t table;
5354     uint32_t desc;
5355     uint32_t xn;
5356     uint32_t pxn = 0;
5357     int type;
5358     int ap;
5359     int domain = 0;
5360     int domain_prot;
5361     hwaddr phys_addr;
5362     uint32_t dacr;
5363     bool ns;
5364
5365     /* Pagetable walk.  */
5366     /* Lookup l1 descriptor.  */
5367     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
5368         /* Section translation fault if page walk is disabled by PD0 or PD1 */
5369         code = 5;
5370         goto do_fault;
5371     }
5372     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5373     type = (desc & 3);
5374     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
5375         /* Section translation fault, or attempt to use the encoding
5376          * which is Reserved on implementations without PXN.
5377          */
5378         code = 5;
5379         goto do_fault;
5380     }
5381     if ((type == 1) || !(desc & (1 << 18))) {
5382         /* Page or Section.  */
5383         domain = (desc >> 5) & 0x0f;
5384     }
5385     if (regime_el(env, mmu_idx) == 1) {
5386         dacr = env->cp15.dacr_ns;
5387     } else {
5388         dacr = env->cp15.dacr_s;
5389     }
5390     domain_prot = (dacr >> (domain * 2)) & 3;
5391     if (domain_prot == 0 || domain_prot == 2) {
5392         if (type != 1) {
5393             code = 9; /* Section domain fault.  */
5394         } else {
5395             code = 11; /* Page domain fault.  */
5396         }
5397         goto do_fault;
5398     }
5399     if (type != 1) {
5400         if (desc & (1 << 18)) {
5401             /* Supersection.  */
5402             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
5403             *page_size = 0x1000000;
5404         } else {
5405             /* Section.  */
5406             phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
5407             *page_size = 0x100000;
5408         }
5409         ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
5410         xn = desc & (1 << 4);
5411         pxn = desc & 1;
5412         code = 13;
5413         ns = extract32(desc, 19, 1);
5414     } else {
5415         if (arm_feature(env, ARM_FEATURE_PXN)) {
5416             pxn = (desc >> 2) & 1;
5417         }
5418         ns = extract32(desc, 3, 1);
5419         /* Lookup l2 entry.  */
5420         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
5421         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5422         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
5423         switch (desc & 3) {
5424         case 0: /* Page translation fault.  */
5425             code = 7;
5426             goto do_fault;
5427         case 1: /* 64k page.  */
5428             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
5429             xn = desc & (1 << 15);
5430             *page_size = 0x10000;
5431             break;
5432         case 2: case 3: /* 4k page.  */
5433             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
5434             xn = desc & 1;
5435             *page_size = 0x1000;
5436             break;
5437         default:
5438             /* Never happens, but compiler isn't smart enough to tell.  */
5439             abort();
5440         }
5441         code = 15;
5442     }
5443     if (domain_prot == 3) {
5444         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
5445     } else {
5446         if (pxn && !regime_is_user(env, mmu_idx)) {
5447             xn = 1;
5448         }
5449         if (xn && access_type == 2)
5450             goto do_fault;
5451
5452         if (arm_feature(env, ARM_FEATURE_V6K) &&
5453                 (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
5454             /* The simplified model uses AP[0] as an access control bit.  */
5455             if ((ap & 1) == 0) {
5456                 /* Access flag fault.  */
5457                 code = (code == 15) ? 6 : 3;
5458                 goto do_fault;
5459             }
5460             *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
5461         } else {
5462             *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
5463         }
5464         if (*prot && !xn) {
5465             *prot |= PAGE_EXEC;
5466         }
5467         if (!(*prot & (1 << access_type))) {
5468             /* Access permission fault.  */
5469             goto do_fault;
5470         }
5471     }
5472     if (ns) {
5473         /* The NS bit will (as required by the architecture) have no effect if
5474          * the CPU doesn't support TZ or this is a non-secure translation
5475          * regime, because the attribute will already be non-secure.
5476          */
5477         attrs->secure = false;
5478     }
5479     *phys_ptr = phys_addr;
5480     return 0;
5481 do_fault:
5482     return code | (domain << 4);
5483 }
5484
5485 /* Fault type for long-descriptor MMU fault reporting; this corresponds
5486  * to bits [5..2] in the STATUS field in long-format DFSR/IFSR.
5487  */
5488 typedef enum {
5489     translation_fault = 1,
5490     access_fault = 2,
5491     permission_fault = 3,
5492 } MMUFaultType;
5493
5494 static int get_phys_addr_lpae(CPUARMState *env, target_ulong address,
5495                               int access_type, ARMMMUIdx mmu_idx,
5496                               hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
5497                               target_ulong *page_size_ptr)
5498 {
5499     CPUState *cs = CPU(arm_env_get_cpu(env));
5500     /* Read an LPAE long-descriptor translation table. */
5501     MMUFaultType fault_type = translation_fault;
5502     uint32_t level = 1;
5503     uint32_t epd;
5504     int32_t tsz;
5505     uint32_t tg;
5506     uint64_t ttbr;
5507     int ttbr_select;
5508     hwaddr descaddr, descmask;
5509     uint32_t tableattrs;
5510     target_ulong page_size;
5511     uint32_t attrs;
5512     int32_t granule_sz = 9;
5513     int32_t va_size = 32;
5514     int32_t tbi = 0;
5515     TCR *tcr = regime_tcr(env, mmu_idx);
5516     int ap, ns, xn, pxn;
5517     uint32_t el = regime_el(env, mmu_idx);
5518     bool ttbr1_valid = true;
5519
5520     /* TODO:
5521      * This code does not handle the different format TCR for VTCR_EL2.
5522      * This code also does not support shareability levels.
5523      * Attribute and permission bit handling should also be checked when adding
5524      * support for those page table walks.
5525      */
5526     if (arm_el_is_aa64(env, el)) {
5527         va_size = 64;
5528         if (el > 1) {
5529             tbi = extract64(tcr->raw_tcr, 20, 1);
5530         } else {
5531             if (extract64(address, 55, 1)) {
5532                 tbi = extract64(tcr->raw_tcr, 38, 1);
5533             } else {
5534                 tbi = extract64(tcr->raw_tcr, 37, 1);
5535             }
5536         }
5537         tbi *= 8;
5538
5539         /* If we are in 64-bit EL2 or EL3 then there is no TTBR1, so mark it
5540          * invalid.
5541          */
5542         if (el > 1) {
5543             ttbr1_valid = false;
5544         }
5545     }
5546
5547     /* Determine whether this address is in the region controlled by
5548      * TTBR0 or TTBR1 (or if it is in neither region and should fault).
5549      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
5550      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
5551      */
5552     uint32_t t0sz = extract32(tcr->raw_tcr, 0, 6);
5553     if (va_size == 64) {
5554         t0sz = MIN(t0sz, 39);
5555         t0sz = MAX(t0sz, 16);
5556     }
5557     uint32_t t1sz = extract32(tcr->raw_tcr, 16, 6);
5558     if (va_size == 64) {
5559         t1sz = MIN(t1sz, 39);
5560         t1sz = MAX(t1sz, 16);
5561     }
5562     if (t0sz && !extract64(address, va_size - t0sz, t0sz - tbi)) {
5563         /* there is a ttbr0 region and we are in it (high bits all zero) */
5564         ttbr_select = 0;
5565     } else if (ttbr1_valid && t1sz &&
5566                !extract64(~address, va_size - t1sz, t1sz - tbi)) {
5567         /* there is a ttbr1 region and we are in it (high bits all one) */
5568         ttbr_select = 1;
5569     } else if (!t0sz) {
5570         /* ttbr0 region is "everything not in the ttbr1 region" */
5571         ttbr_select = 0;
5572     } else if (!t1sz && ttbr1_valid) {
5573         /* ttbr1 region is "everything not in the ttbr0 region" */
5574         ttbr_select = 1;
5575     } else {
5576         /* in the gap between the two regions, this is a Translation fault */
5577         fault_type = translation_fault;
5578         goto do_fault;
5579     }
5580
5581     /* Note that QEMU ignores shareability and cacheability attributes,
5582      * so we don't need to do anything with the SH, ORGN, IRGN fields
5583      * in the TTBCR.  Similarly, TTBCR:A1 selects whether we get the
5584      * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
5585      * implement any ASID-like capability so we can ignore it (instead
5586      * we will always flush the TLB any time the ASID is changed).
5587      */
5588     if (ttbr_select == 0) {
5589         ttbr = regime_ttbr(env, mmu_idx, 0);
5590         epd = extract32(tcr->raw_tcr, 7, 1);
5591         tsz = t0sz;
5592
5593         tg = extract32(tcr->raw_tcr, 14, 2);
5594         if (tg == 1) { /* 64KB pages */
5595             granule_sz = 13;
5596         }
5597         if (tg == 2) { /* 16KB pages */
5598             granule_sz = 11;
5599         }
5600     } else {
5601         /* We should only be here if TTBR1 is valid */
5602         assert(ttbr1_valid);
5603
5604         ttbr = regime_ttbr(env, mmu_idx, 1);
5605         epd = extract32(tcr->raw_tcr, 23, 1);
5606         tsz = t1sz;
5607
5608         tg = extract32(tcr->raw_tcr, 30, 2);
5609         if (tg == 3)  { /* 64KB pages */
5610             granule_sz = 13;
5611         }
5612         if (tg == 1) { /* 16KB pages */
5613             granule_sz = 11;
5614         }
5615     }
5616
5617     /* Here we should have set up all the parameters for the translation:
5618      * va_size, ttbr, epd, tsz, granule_sz, tbi
5619      */
5620
5621     if (epd) {
5622         /* Translation table walk disabled => Translation fault on TLB miss
5623          * Note: This is always 0 on 64-bit EL2 and EL3.
5624          */
5625         goto do_fault;
5626     }
5627
5628     /* The starting level depends on the virtual address size (which can be
5629      * up to 48 bits) and the translation granule size. It indicates the number
5630      * of strides (granule_sz bits at a time) needed to consume the bits
5631      * of the input address. In the pseudocode this is:
5632      *  level = 4 - RoundUp((inputsize - grainsize) / stride)
5633      * where their 'inputsize' is our 'va_size - tsz', 'grainsize' is
5634      * our 'granule_sz + 3' and 'stride' is our 'granule_sz'.
5635      * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
5636      *     = 4 - (va_size - tsz - granule_sz - 3 + granule_sz - 1) / granule_sz
5637      *     = 4 - (va_size - tsz - 4) / granule_sz;
5638      */
5639     level = 4 - (va_size - tsz - 4) / granule_sz;
5640
5641     /* Clear the vaddr bits which aren't part of the within-region address,
5642      * so that we don't have to special case things when calculating the
5643      * first descriptor address.
5644      */
5645     if (tsz) {
5646         address &= (1ULL << (va_size - tsz)) - 1;
5647     }
5648
5649     descmask = (1ULL << (granule_sz + 3)) - 1;
5650
5651     /* Now we can extract the actual base address from the TTBR */
5652     descaddr = extract64(ttbr, 0, 48);
5653     descaddr &= ~((1ULL << (va_size - tsz - (granule_sz * (4 - level)))) - 1);
5654
5655     /* Secure accesses start with the page table in secure memory and
5656      * can be downgraded to non-secure at any step. Non-secure accesses
5657      * remain non-secure. We implement this by just ORing in the NSTable/NS
5658      * bits at each step.
5659      */
5660     tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4);
5661     for (;;) {
5662         uint64_t descriptor;
5663         bool nstable;
5664
5665         descaddr |= (address >> (granule_sz * (4 - level))) & descmask;
5666         descaddr &= ~7ULL;
5667         nstable = extract32(tableattrs, 4, 1);
5668         descriptor = arm_ldq_ptw(cs, descaddr, !nstable);
5669         if (!(descriptor & 1) ||
5670             (!(descriptor & 2) && (level == 3))) {
5671             /* Invalid, or the Reserved level 3 encoding */
5672             goto do_fault;
5673         }
5674         descaddr = descriptor & 0xfffffff000ULL;
5675
5676         if ((descriptor & 2) && (level < 3)) {
5677             /* Table entry. The top five bits are attributes which  may
5678              * propagate down through lower levels of the table (and
5679              * which are all arranged so that 0 means "no effect", so
5680              * we can gather them up by ORing in the bits at each level).
5681              */
5682             tableattrs |= extract64(descriptor, 59, 5);
5683             level++;
5684             continue;
5685         }
5686         /* Block entry at level 1 or 2, or page entry at level 3.
5687          * These are basically the same thing, although the number
5688          * of bits we pull in from the vaddr varies.
5689          */
5690         page_size = (1ULL << ((granule_sz * (4 - level)) + 3));
5691         descaddr |= (address & (page_size - 1));
5692         /* Extract attributes from the descriptor and merge with table attrs */
5693         attrs = extract64(descriptor, 2, 10)
5694             | (extract64(descriptor, 52, 12) << 10);
5695         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
5696         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
5697         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
5698          * means "force PL1 access only", which means forcing AP[1] to 0.
5699          */
5700         if (extract32(tableattrs, 2, 1)) {
5701             attrs &= ~(1 << 4);
5702         }
5703         attrs |= nstable << 3; /* NS */
5704         break;
5705     }
5706     /* Here descaddr is the final physical address, and attributes
5707      * are all in attrs.
5708      */
5709     fault_type = access_fault;
5710     if ((attrs & (1 << 8)) == 0) {
5711         /* Access flag */
5712         goto do_fault;
5713     }
5714
5715     ap = extract32(attrs, 4, 2);
5716     ns = extract32(attrs, 3, 1);
5717     xn = extract32(attrs, 12, 1);
5718     pxn = extract32(attrs, 11, 1);
5719
5720     *prot = get_S1prot(env, mmu_idx, va_size == 64, ap, ns, xn, pxn);
5721
5722     fault_type = permission_fault;
5723     if (!(*prot & (1 << access_type))) {
5724         goto do_fault;
5725     }
5726
5727     if (ns) {
5728         /* The NS bit will (as required by the architecture) have no effect if
5729          * the CPU doesn't support TZ or this is a non-secure translation
5730          * regime, because the attribute will already be non-secure.
5731          */
5732         txattrs->secure = false;
5733     }
5734     *phys_ptr = descaddr;
5735     *page_size_ptr = page_size;
5736     return 0;
5737
5738 do_fault:
5739     /* Long-descriptor format IFSR/DFSR value */
5740     return (1 << 9) | (fault_type << 2) | level;
5741 }
5742
5743 static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
5744                              int access_type, ARMMMUIdx mmu_idx,
5745                              hwaddr *phys_ptr, int *prot)
5746 {
5747     int n;
5748     uint32_t mask;
5749     uint32_t base;
5750     bool is_user = regime_is_user(env, mmu_idx);
5751
5752     *phys_ptr = address;
5753     for (n = 7; n >= 0; n--) {
5754         base = env->cp15.c6_region[n];
5755         if ((base & 1) == 0) {
5756             continue;
5757         }
5758         mask = 1 << ((base >> 1) & 0x1f);
5759         /* Keep this shift separate from the above to avoid an
5760            (undefined) << 32.  */
5761         mask = (mask << 1) - 1;
5762         if (((base ^ address) & ~mask) == 0) {
5763             break;
5764         }
5765     }
5766     if (n < 0) {
5767         return 2;
5768     }
5769
5770     if (access_type == 2) {
5771         mask = env->cp15.pmsav5_insn_ap;
5772     } else {
5773         mask = env->cp15.pmsav5_data_ap;
5774     }
5775     mask = (mask >> (n * 4)) & 0xf;
5776     switch (mask) {
5777     case 0:
5778         return 1;
5779     case 1:
5780         if (is_user) {
5781             return 1;
5782         }
5783         *prot = PAGE_READ | PAGE_WRITE;
5784         break;
5785     case 2:
5786         *prot = PAGE_READ;
5787         if (!is_user) {
5788             *prot |= PAGE_WRITE;
5789         }
5790         break;
5791     case 3:
5792         *prot = PAGE_READ | PAGE_WRITE;
5793         break;
5794     case 5:
5795         if (is_user) {
5796             return 1;
5797         }
5798         *prot = PAGE_READ;
5799         break;
5800     case 6:
5801         *prot = PAGE_READ;
5802         break;
5803     default:
5804         /* Bad permission.  */
5805         return 1;
5806     }
5807     *prot |= PAGE_EXEC;
5808     return 0;
5809 }
5810
5811 /* get_phys_addr - get the physical address for this virtual address
5812  *
5813  * Find the physical address corresponding to the given virtual address,
5814  * by doing a translation table walk on MMU based systems or using the
5815  * MPU state on MPU based systems.
5816  *
5817  * Returns 0 if the translation was successful. Otherwise, phys_ptr, attrs,
5818  * prot and page_size may not be filled in, and the return value provides
5819  * information on why the translation aborted, in the format of a
5820  * DFSR/IFSR fault register, with the following caveats:
5821  *  * we honour the short vs long DFSR format differences.
5822  *  * the WnR bit is never set (the caller must do this).
5823  *  * for MPU based systems we don't bother to return a full FSR format
5824  *    value.
5825  *
5826  * @env: CPUARMState
5827  * @address: virtual address to get physical address for
5828  * @access_type: 0 for read, 1 for write, 2 for execute
5829  * @mmu_idx: MMU index indicating required translation regime
5830  * @phys_ptr: set to the physical address corresponding to the virtual address
5831  * @attrs: set to the memory transaction attributes to use
5832  * @prot: set to the permissions for the page containing phys_ptr
5833  * @page_size: set to the size of the page containing phys_ptr
5834  */
5835 static inline int get_phys_addr(CPUARMState *env, target_ulong address,
5836                                 int access_type, ARMMMUIdx mmu_idx,
5837                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
5838                                 target_ulong *page_size)
5839 {
5840     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
5841         /* TODO: when we support EL2 we should here call ourselves recursively
5842          * to do the stage 1 and then stage 2 translations. The arm_ld*_ptw
5843          * functions will also need changing to perform ARMMMUIdx_S2NS loads
5844          * rather than direct physical memory loads when appropriate.
5845          * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
5846          */
5847         assert(!arm_feature(env, ARM_FEATURE_EL2));
5848         mmu_idx += ARMMMUIdx_S1NSE0;
5849     }
5850
5851     /* The page table entries may downgrade secure to non-secure, but
5852      * cannot upgrade an non-secure translation regime's attributes
5853      * to secure.
5854      */
5855     attrs->secure = regime_is_secure(env, mmu_idx);
5856     attrs->user = regime_is_user(env, mmu_idx);
5857
5858     /* Fast Context Switch Extension. This doesn't exist at all in v8.
5859      * In v7 and earlier it affects all stage 1 translations.
5860      */
5861     if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS
5862         && !arm_feature(env, ARM_FEATURE_V8)) {
5863         if (regime_el(env, mmu_idx) == 3) {
5864             address += env->cp15.fcseidr_s;
5865         } else {
5866             address += env->cp15.fcseidr_ns;
5867         }
5868     }
5869
5870     if (regime_translation_disabled(env, mmu_idx)) {
5871         /* MMU/MPU disabled.  */
5872         *phys_ptr = address;
5873         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
5874         *page_size = TARGET_PAGE_SIZE;
5875         return 0;
5876     }
5877
5878     if (arm_feature(env, ARM_FEATURE_MPU)) {
5879         *page_size = TARGET_PAGE_SIZE;
5880         return get_phys_addr_mpu(env, address, access_type, mmu_idx, phys_ptr,
5881                                  prot);
5882     }
5883
5884     if (regime_using_lpae_format(env, mmu_idx)) {
5885         return get_phys_addr_lpae(env, address, access_type, mmu_idx, phys_ptr,
5886                                   attrs, prot, page_size);
5887     } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
5888         return get_phys_addr_v6(env, address, access_type, mmu_idx, phys_ptr,
5889                                 attrs, prot, page_size);
5890     } else {
5891         return get_phys_addr_v5(env, address, access_type, mmu_idx, phys_ptr,
5892                                 prot, page_size);
5893     }
5894 }
5895
5896 /* Walk the page table and (if the mapping exists) add the page
5897  * to the TLB. Return 0 on success, or an ARM DFSR/IFSR fault
5898  * register format value on failure.
5899  */
5900 int arm_tlb_fill(CPUState *cs, vaddr address,
5901                  int access_type, int mmu_idx)
5902 {
5903     ARMCPU *cpu = ARM_CPU(cs);
5904     CPUARMState *env = &cpu->env;
5905     hwaddr phys_addr;
5906     target_ulong page_size;
5907     int prot;
5908     int ret;
5909     MemTxAttrs attrs = {};
5910
5911     ret = get_phys_addr(env, address, access_type, mmu_idx, &phys_addr,
5912                         &attrs, &prot, &page_size);
5913     if (ret == 0) {
5914         /* Map a single [sub]page.  */
5915         phys_addr &= TARGET_PAGE_MASK;
5916         address &= TARGET_PAGE_MASK;
5917         tlb_set_page_with_attrs(cs, address, phys_addr, attrs,
5918                                 prot, mmu_idx, page_size);
5919         return 0;
5920     }
5921
5922     return ret;
5923 }
5924
5925 hwaddr arm_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
5926 {
5927     ARMCPU *cpu = ARM_CPU(cs);
5928     CPUARMState *env = &cpu->env;
5929     hwaddr phys_addr;
5930     target_ulong page_size;
5931     int prot;
5932     int ret;
5933     MemTxAttrs attrs = {};
5934
5935     ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env), &phys_addr,
5936                         &attrs, &prot, &page_size);
5937
5938     if (ret != 0) {
5939         return -1;
5940     }
5941
5942     return phys_addr;
5943 }
5944
5945 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
5946 {
5947     if ((env->uncached_cpsr & CPSR_M) == mode) {
5948         env->regs[13] = val;
5949     } else {
5950         env->banked_r13[bank_number(mode)] = val;
5951     }
5952 }
5953
5954 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
5955 {
5956     if ((env->uncached_cpsr & CPSR_M) == mode) {
5957         return env->regs[13];
5958     } else {
5959         return env->banked_r13[bank_number(mode)];
5960     }
5961 }
5962
5963 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
5964 {
5965     ARMCPU *cpu = arm_env_get_cpu(env);
5966
5967     switch (reg) {
5968     case 0: /* APSR */
5969         return xpsr_read(env) & 0xf8000000;
5970     case 1: /* IAPSR */
5971         return xpsr_read(env) & 0xf80001ff;
5972     case 2: /* EAPSR */
5973         return xpsr_read(env) & 0xff00fc00;
5974     case 3: /* xPSR */
5975         return xpsr_read(env) & 0xff00fdff;
5976     case 5: /* IPSR */
5977         return xpsr_read(env) & 0x000001ff;
5978     case 6: /* EPSR */
5979         return xpsr_read(env) & 0x0700fc00;
5980     case 7: /* IEPSR */
5981         return xpsr_read(env) & 0x0700edff;
5982     case 8: /* MSP */
5983         return env->v7m.current_sp ? env->v7m.other_sp : env->regs[13];
5984     case 9: /* PSP */
5985         return env->v7m.current_sp ? env->regs[13] : env->v7m.other_sp;
5986     case 16: /* PRIMASK */
5987         return (env->daif & PSTATE_I) != 0;
5988     case 17: /* BASEPRI */
5989     case 18: /* BASEPRI_MAX */
5990         return env->v7m.basepri;
5991     case 19: /* FAULTMASK */
5992         return (env->daif & PSTATE_F) != 0;
5993     case 20: /* CONTROL */
5994         return env->v7m.control;
5995     default:
5996         /* ??? For debugging only.  */
5997         cpu_abort(CPU(cpu), "Unimplemented system register read (%d)\n", reg);
5998         return 0;
5999     }
6000 }
6001
6002 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
6003 {
6004     ARMCPU *cpu = arm_env_get_cpu(env);
6005
6006     switch (reg) {
6007     case 0: /* APSR */
6008         xpsr_write(env, val, 0xf8000000);
6009         break;
6010     case 1: /* IAPSR */
6011         xpsr_write(env, val, 0xf8000000);
6012         break;
6013     case 2: /* EAPSR */
6014         xpsr_write(env, val, 0xfe00fc00);
6015         break;
6016     case 3: /* xPSR */
6017         xpsr_write(env, val, 0xfe00fc00);
6018         break;
6019     case 5: /* IPSR */
6020         /* IPSR bits are readonly.  */
6021         break;
6022     case 6: /* EPSR */
6023         xpsr_write(env, val, 0x0600fc00);
6024         break;
6025     case 7: /* IEPSR */
6026         xpsr_write(env, val, 0x0600fc00);
6027         break;
6028     case 8: /* MSP */
6029         if (env->v7m.current_sp)
6030             env->v7m.other_sp = val;
6031         else
6032             env->regs[13] = val;
6033         break;
6034     case 9: /* PSP */
6035         if (env->v7m.current_sp)
6036             env->regs[13] = val;
6037         else
6038             env->v7m.other_sp = val;
6039         break;
6040     case 16: /* PRIMASK */
6041         if (val & 1) {
6042             env->daif |= PSTATE_I;
6043         } else {
6044             env->daif &= ~PSTATE_I;
6045         }
6046         break;
6047     case 17: /* BASEPRI */
6048         env->v7m.basepri = val & 0xff;
6049         break;
6050     case 18: /* BASEPRI_MAX */
6051         val &= 0xff;
6052         if (val != 0 && (val < env->v7m.basepri || env->v7m.basepri == 0))
6053             env->v7m.basepri = val;
6054         break;
6055     case 19: /* FAULTMASK */
6056         if (val & 1) {
6057             env->daif |= PSTATE_F;
6058         } else {
6059             env->daif &= ~PSTATE_F;
6060         }
6061         break;
6062     case 20: /* CONTROL */
6063         env->v7m.control = val & 3;
6064         switch_v7m_sp(env, (val & 2) != 0);
6065         break;
6066     default:
6067         /* ??? For debugging only.  */
6068         cpu_abort(CPU(cpu), "Unimplemented system register write (%d)\n", reg);
6069         return;
6070     }
6071 }
6072
6073 #endif
6074
6075 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
6076 {
6077     /* Implement DC ZVA, which zeroes a fixed-length block of memory.
6078      * Note that we do not implement the (architecturally mandated)
6079      * alignment fault for attempts to use this on Device memory
6080      * (which matches the usual QEMU behaviour of not implementing either
6081      * alignment faults or any memory attribute handling).
6082      */
6083
6084     ARMCPU *cpu = arm_env_get_cpu(env);
6085     uint64_t blocklen = 4 << cpu->dcz_blocksize;
6086     uint64_t vaddr = vaddr_in & ~(blocklen - 1);
6087
6088 #ifndef CONFIG_USER_ONLY
6089     {
6090         /* Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than
6091          * the block size so we might have to do more than one TLB lookup.
6092          * We know that in fact for any v8 CPU the page size is at least 4K
6093          * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only
6094          * 1K as an artefact of legacy v5 subpage support being present in the
6095          * same QEMU executable.
6096          */
6097         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
6098         void *hostaddr[maxidx];
6099         int try, i;
6100         unsigned mmu_idx = cpu_mmu_index(env);
6101         TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
6102
6103         for (try = 0; try < 2; try++) {
6104
6105             for (i = 0; i < maxidx; i++) {
6106                 hostaddr[i] = tlb_vaddr_to_host(env,
6107                                                 vaddr + TARGET_PAGE_SIZE * i,
6108                                                 1, mmu_idx);
6109                 if (!hostaddr[i]) {
6110                     break;
6111                 }
6112             }
6113             if (i == maxidx) {
6114                 /* If it's all in the TLB it's fair game for just writing to;
6115                  * we know we don't need to update dirty status, etc.
6116                  */
6117                 for (i = 0; i < maxidx - 1; i++) {
6118                     memset(hostaddr[i], 0, TARGET_PAGE_SIZE);
6119                 }
6120                 memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE));
6121                 return;
6122             }
6123             /* OK, try a store and see if we can populate the tlb. This
6124              * might cause an exception if the memory isn't writable,
6125              * in which case we will longjmp out of here. We must for
6126              * this purpose use the actual register value passed to us
6127              * so that we get the fault address right.
6128              */
6129             helper_ret_stb_mmu(env, vaddr_in, 0, oi, GETRA());
6130             /* Now we can populate the other TLB entries, if any */
6131             for (i = 0; i < maxidx; i++) {
6132                 uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
6133                 if (va != (vaddr_in & TARGET_PAGE_MASK)) {
6134                     helper_ret_stb_mmu(env, va, 0, oi, GETRA());
6135                 }
6136             }
6137         }
6138
6139         /* Slow path (probably attempt to do this to an I/O device or
6140          * similar, or clearing of a block of code we have translations
6141          * cached for). Just do a series of byte writes as the architecture
6142          * demands. It's not worth trying to use a cpu_physical_memory_map(),
6143          * memset(), unmap() sequence here because:
6144          *  + we'd need to account for the blocksize being larger than a page
6145          *  + the direct-RAM access case is almost always going to be dealt
6146          *    with in the fastpath code above, so there's no speed benefit
6147          *  + we would have to deal with the map returning NULL because the
6148          *    bounce buffer was in use
6149          */
6150         for (i = 0; i < blocklen; i++) {
6151             helper_ret_stb_mmu(env, vaddr + i, 0, oi, GETRA());
6152         }
6153     }
6154 #else
6155     memset(g2h(vaddr), 0, blocklen);
6156 #endif
6157 }
6158
6159 /* Note that signed overflow is undefined in C.  The following routines are
6160    careful to use unsigned types where modulo arithmetic is required.
6161    Failure to do so _will_ break on newer gcc.  */
6162
6163 /* Signed saturating arithmetic.  */
6164
6165 /* Perform 16-bit signed saturating addition.  */
6166 static inline uint16_t add16_sat(uint16_t a, uint16_t b)
6167 {
6168     uint16_t res;
6169
6170     res = a + b;
6171     if (((res ^ a) & 0x8000) && !((a ^ b) & 0x8000)) {
6172         if (a & 0x8000)
6173             res = 0x8000;
6174         else
6175             res = 0x7fff;
6176     }
6177     return res;
6178 }
6179
6180 /* Perform 8-bit signed saturating addition.  */
6181 static inline uint8_t add8_sat(uint8_t a, uint8_t b)
6182 {
6183     uint8_t res;
6184
6185     res = a + b;
6186     if (((res ^ a) & 0x80) && !((a ^ b) & 0x80)) {
6187         if (a & 0x80)
6188             res = 0x80;
6189         else
6190             res = 0x7f;
6191     }
6192     return res;
6193 }
6194
6195 /* Perform 16-bit signed saturating subtraction.  */
6196 static inline uint16_t sub16_sat(uint16_t a, uint16_t b)
6197 {
6198     uint16_t res;
6199
6200     res = a - b;
6201     if (((res ^ a) & 0x8000) && ((a ^ b) & 0x8000)) {
6202         if (a & 0x8000)
6203             res = 0x8000;
6204         else
6205             res = 0x7fff;
6206     }
6207     return res;
6208 }
6209
6210 /* Perform 8-bit signed saturating subtraction.  */
6211 static inline uint8_t sub8_sat(uint8_t a, uint8_t b)
6212 {
6213     uint8_t res;
6214
6215     res = a - b;
6216     if (((res ^ a) & 0x80) && ((a ^ b) & 0x80)) {
6217         if (a & 0x80)
6218             res = 0x80;
6219         else
6220             res = 0x7f;
6221     }
6222     return res;
6223 }
6224
6225 #define ADD16(a, b, n) RESULT(add16_sat(a, b), n, 16);
6226 #define SUB16(a, b, n) RESULT(sub16_sat(a, b), n, 16);
6227 #define ADD8(a, b, n)  RESULT(add8_sat(a, b), n, 8);
6228 #define SUB8(a, b, n)  RESULT(sub8_sat(a, b), n, 8);
6229 #define PFX q
6230
6231 #include "op_addsub.h"
6232
6233 /* Unsigned saturating arithmetic.  */
6234 static inline uint16_t add16_usat(uint16_t a, uint16_t b)
6235 {
6236     uint16_t res;
6237     res = a + b;
6238     if (res < a)
6239         res = 0xffff;
6240     return res;
6241 }
6242
6243 static inline uint16_t sub16_usat(uint16_t a, uint16_t b)
6244 {
6245     if (a > b)
6246         return a - b;
6247     else
6248         return 0;
6249 }
6250
6251 static inline uint8_t add8_usat(uint8_t a, uint8_t b)
6252 {
6253     uint8_t res;
6254     res = a + b;
6255     if (res < a)
6256         res = 0xff;
6257     return res;
6258 }
6259
6260 static inline uint8_t sub8_usat(uint8_t a, uint8_t b)
6261 {
6262     if (a > b)
6263         return a - b;
6264     else
6265         return 0;
6266 }
6267
6268 #define ADD16(a, b, n) RESULT(add16_usat(a, b), n, 16);
6269 #define SUB16(a, b, n) RESULT(sub16_usat(a, b), n, 16);
6270 #define ADD8(a, b, n)  RESULT(add8_usat(a, b), n, 8);
6271 #define SUB8(a, b, n)  RESULT(sub8_usat(a, b), n, 8);
6272 #define PFX uq
6273
6274 #include "op_addsub.h"
6275
6276 /* Signed modulo arithmetic.  */
6277 #define SARITH16(a, b, n, op) do { \
6278     int32_t sum; \
6279     sum = (int32_t)(int16_t)(a) op (int32_t)(int16_t)(b); \
6280     RESULT(sum, n, 16); \
6281     if (sum >= 0) \
6282         ge |= 3 << (n * 2); \
6283     } while(0)
6284
6285 #define SARITH8(a, b, n, op) do { \
6286     int32_t sum; \
6287     sum = (int32_t)(int8_t)(a) op (int32_t)(int8_t)(b); \
6288     RESULT(sum, n, 8); \
6289     if (sum >= 0) \
6290         ge |= 1 << n; \
6291     } while(0)
6292
6293
6294 #define ADD16(a, b, n) SARITH16(a, b, n, +)
6295 #define SUB16(a, b, n) SARITH16(a, b, n, -)
6296 #define ADD8(a, b, n)  SARITH8(a, b, n, +)
6297 #define SUB8(a, b, n)  SARITH8(a, b, n, -)
6298 #define PFX s
6299 #define ARITH_GE
6300
6301 #include "op_addsub.h"
6302
6303 /* Unsigned modulo arithmetic.  */
6304 #define ADD16(a, b, n) do { \
6305     uint32_t sum; \
6306     sum = (uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b); \
6307     RESULT(sum, n, 16); \
6308     if ((sum >> 16) == 1) \
6309         ge |= 3 << (n * 2); \
6310     } while(0)
6311
6312 #define ADD8(a, b, n) do { \
6313     uint32_t sum; \
6314     sum = (uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b); \
6315     RESULT(sum, n, 8); \
6316     if ((sum >> 8) == 1) \
6317         ge |= 1 << n; \
6318     } while(0)
6319
6320 #define SUB16(a, b, n) do { \
6321     uint32_t sum; \
6322     sum = (uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b); \
6323     RESULT(sum, n, 16); \
6324     if ((sum >> 16) == 0) \
6325         ge |= 3 << (n * 2); \
6326     } while(0)
6327
6328 #define SUB8(a, b, n) do { \
6329     uint32_t sum; \
6330     sum = (uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b); \
6331     RESULT(sum, n, 8); \
6332     if ((sum >> 8) == 0) \
6333         ge |= 1 << n; \
6334     } while(0)
6335
6336 #define PFX u
6337 #define ARITH_GE
6338
6339 #include "op_addsub.h"
6340
6341 /* Halved signed arithmetic.  */
6342 #define ADD16(a, b, n) \
6343   RESULT(((int32_t)(int16_t)(a) + (int32_t)(int16_t)(b)) >> 1, n, 16)
6344 #define SUB16(a, b, n) \
6345   RESULT(((int32_t)(int16_t)(a) - (int32_t)(int16_t)(b)) >> 1, n, 16)
6346 #define ADD8(a, b, n) \
6347   RESULT(((int32_t)(int8_t)(a) + (int32_t)(int8_t)(b)) >> 1, n, 8)
6348 #define SUB8(a, b, n) \
6349   RESULT(((int32_t)(int8_t)(a) - (int32_t)(int8_t)(b)) >> 1, n, 8)
6350 #define PFX sh
6351
6352 #include "op_addsub.h"
6353
6354 /* Halved unsigned arithmetic.  */
6355 #define ADD16(a, b, n) \
6356   RESULT(((uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b)) >> 1, n, 16)
6357 #define SUB16(a, b, n) \
6358   RESULT(((uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b)) >> 1, n, 16)
6359 #define ADD8(a, b, n) \
6360   RESULT(((uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b)) >> 1, n, 8)
6361 #define SUB8(a, b, n) \
6362   RESULT(((uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b)) >> 1, n, 8)
6363 #define PFX uh
6364
6365 #include "op_addsub.h"
6366
6367 static inline uint8_t do_usad(uint8_t a, uint8_t b)
6368 {
6369     if (a > b)
6370         return a - b;
6371     else
6372         return b - a;
6373 }
6374
6375 /* Unsigned sum of absolute byte differences.  */
6376 uint32_t HELPER(usad8)(uint32_t a, uint32_t b)
6377 {
6378     uint32_t sum;
6379     sum = do_usad(a, b);
6380     sum += do_usad(a >> 8, b >> 8);
6381     sum += do_usad(a >> 16, b >>16);
6382     sum += do_usad(a >> 24, b >> 24);
6383     return sum;
6384 }
6385
6386 /* For ARMv6 SEL instruction.  */
6387 uint32_t HELPER(sel_flags)(uint32_t flags, uint32_t a, uint32_t b)
6388 {
6389     uint32_t mask;
6390
6391     mask = 0;
6392     if (flags & 1)
6393         mask |= 0xff;
6394     if (flags & 2)
6395         mask |= 0xff00;
6396     if (flags & 4)
6397         mask |= 0xff0000;
6398     if (flags & 8)
6399         mask |= 0xff000000;
6400     return (a & mask) | (b & ~mask);
6401 }
6402
6403 /* VFP support.  We follow the convention used for VFP instructions:
6404    Single precision routines have a "s" suffix, double precision a
6405    "d" suffix.  */
6406
6407 /* Convert host exception flags to vfp form.  */
6408 static inline int vfp_exceptbits_from_host(int host_bits)
6409 {
6410     int target_bits = 0;
6411
6412     if (host_bits & float_flag_invalid)
6413         target_bits |= 1;
6414     if (host_bits & float_flag_divbyzero)
6415         target_bits |= 2;
6416     if (host_bits & float_flag_overflow)
6417         target_bits |= 4;
6418     if (host_bits & (float_flag_underflow | float_flag_output_denormal))
6419         target_bits |= 8;
6420     if (host_bits & float_flag_inexact)
6421         target_bits |= 0x10;
6422     if (host_bits & float_flag_input_denormal)
6423         target_bits |= 0x80;
6424     return target_bits;
6425 }
6426
6427 uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env)
6428 {
6429     int i;
6430     uint32_t fpscr;
6431
6432     fpscr = (env->vfp.xregs[ARM_VFP_FPSCR] & 0xffc8ffff)
6433             | (env->vfp.vec_len << 16)
6434             | (env->vfp.vec_stride << 20);
6435     i = get_float_exception_flags(&env->vfp.fp_status);
6436     i |= get_float_exception_flags(&env->vfp.standard_fp_status);
6437     fpscr |= vfp_exceptbits_from_host(i);
6438     return fpscr;
6439 }
6440
6441 uint32_t vfp_get_fpscr(CPUARMState *env)
6442 {
6443     return HELPER(vfp_get_fpscr)(env);
6444 }
6445
6446 /* Convert vfp exception flags to target form.  */
6447 static inline int vfp_exceptbits_to_host(int target_bits)
6448 {
6449     int host_bits = 0;
6450
6451     if (target_bits & 1)
6452         host_bits |= float_flag_invalid;
6453     if (target_bits & 2)
6454         host_bits |= float_flag_divbyzero;
6455     if (target_bits & 4)
6456         host_bits |= float_flag_overflow;
6457     if (target_bits & 8)
6458         host_bits |= float_flag_underflow;
6459     if (target_bits & 0x10)
6460         host_bits |= float_flag_inexact;
6461     if (target_bits & 0x80)
6462         host_bits |= float_flag_input_denormal;
6463     return host_bits;
6464 }
6465
6466 void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val)
6467 {
6468     int i;
6469     uint32_t changed;
6470
6471     changed = env->vfp.xregs[ARM_VFP_FPSCR];
6472     env->vfp.xregs[ARM_VFP_FPSCR] = (val & 0xffc8ffff);
6473     env->vfp.vec_len = (val >> 16) & 7;
6474     env->vfp.vec_stride = (val >> 20) & 3;
6475
6476     changed ^= val;
6477     if (changed & (3 << 22)) {
6478         i = (val >> 22) & 3;
6479         switch (i) {
6480         case FPROUNDING_TIEEVEN:
6481             i = float_round_nearest_even;
6482             break;
6483         case FPROUNDING_POSINF:
6484             i = float_round_up;
6485             break;
6486         case FPROUNDING_NEGINF:
6487             i = float_round_down;
6488             break;
6489         case FPROUNDING_ZERO:
6490             i = float_round_to_zero;
6491             break;
6492         }
6493         set_float_rounding_mode(i, &env->vfp.fp_status);
6494     }
6495     if (changed & (1 << 24)) {
6496         set_flush_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
6497         set_flush_inputs_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
6498     }
6499     if (changed & (1 << 25))
6500         set_default_nan_mode((val & (1 << 25)) != 0, &env->vfp.fp_status);
6501
6502     i = vfp_exceptbits_to_host(val);
6503     set_float_exception_flags(i, &env->vfp.fp_status);
6504     set_float_exception_flags(0, &env->vfp.standard_fp_status);
6505 }
6506
6507 void vfp_set_fpscr(CPUARMState *env, uint32_t val)
6508 {
6509     HELPER(vfp_set_fpscr)(env, val);
6510 }
6511
6512 #define VFP_HELPER(name, p) HELPER(glue(glue(vfp_,name),p))
6513
6514 #define VFP_BINOP(name) \
6515 float32 VFP_HELPER(name, s)(float32 a, float32 b, void *fpstp) \
6516 { \
6517     float_status *fpst = fpstp; \
6518     return float32_ ## name(a, b, fpst); \
6519 } \
6520 float64 VFP_HELPER(name, d)(float64 a, float64 b, void *fpstp) \
6521 { \
6522     float_status *fpst = fpstp; \
6523     return float64_ ## name(a, b, fpst); \
6524 }
6525 VFP_BINOP(add)
6526 VFP_BINOP(sub)
6527 VFP_BINOP(mul)
6528 VFP_BINOP(div)
6529 VFP_BINOP(min)
6530 VFP_BINOP(max)
6531 VFP_BINOP(minnum)
6532 VFP_BINOP(maxnum)
6533 #undef VFP_BINOP
6534
6535 float32 VFP_HELPER(neg, s)(float32 a)
6536 {
6537     return float32_chs(a);
6538 }
6539
6540 float64 VFP_HELPER(neg, d)(float64 a)
6541 {
6542     return float64_chs(a);
6543 }
6544
6545 float32 VFP_HELPER(abs, s)(float32 a)
6546 {
6547     return float32_abs(a);
6548 }
6549
6550 float64 VFP_HELPER(abs, d)(float64 a)
6551 {
6552     return float64_abs(a);
6553 }
6554
6555 float32 VFP_HELPER(sqrt, s)(float32 a, CPUARMState *env)
6556 {
6557     return float32_sqrt(a, &env->vfp.fp_status);
6558 }
6559
6560 float64 VFP_HELPER(sqrt, d)(float64 a, CPUARMState *env)
6561 {
6562     return float64_sqrt(a, &env->vfp.fp_status);
6563 }
6564
6565 /* XXX: check quiet/signaling case */
6566 #define DO_VFP_cmp(p, type) \
6567 void VFP_HELPER(cmp, p)(type a, type b, CPUARMState *env)  \
6568 { \
6569     uint32_t flags; \
6570     switch(type ## _compare_quiet(a, b, &env->vfp.fp_status)) { \
6571     case 0: flags = 0x6; break; \
6572     case -1: flags = 0x8; break; \
6573     case 1: flags = 0x2; break; \
6574     default: case 2: flags = 0x3; break; \
6575     } \
6576     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
6577         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
6578 } \
6579 void VFP_HELPER(cmpe, p)(type a, type b, CPUARMState *env) \
6580 { \
6581     uint32_t flags; \
6582     switch(type ## _compare(a, b, &env->vfp.fp_status)) { \
6583     case 0: flags = 0x6; break; \
6584     case -1: flags = 0x8; break; \
6585     case 1: flags = 0x2; break; \
6586     default: case 2: flags = 0x3; break; \
6587     } \
6588     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
6589         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
6590 }
6591 DO_VFP_cmp(s, float32)
6592 DO_VFP_cmp(d, float64)
6593 #undef DO_VFP_cmp
6594
6595 /* Integer to float and float to integer conversions */
6596
6597 #define CONV_ITOF(name, fsz, sign) \
6598     float##fsz HELPER(name)(uint32_t x, void *fpstp) \
6599 { \
6600     float_status *fpst = fpstp; \
6601     return sign##int32_to_##float##fsz((sign##int32_t)x, fpst); \
6602 }
6603
6604 #define CONV_FTOI(name, fsz, sign, round) \
6605 uint32_t HELPER(name)(float##fsz x, void *fpstp) \
6606 { \
6607     float_status *fpst = fpstp; \
6608     if (float##fsz##_is_any_nan(x)) { \
6609         float_raise(float_flag_invalid, fpst); \
6610         return 0; \
6611     } \
6612     return float##fsz##_to_##sign##int32##round(x, fpst); \
6613 }
6614
6615 #define FLOAT_CONVS(name, p, fsz, sign) \
6616 CONV_ITOF(vfp_##name##to##p, fsz, sign) \
6617 CONV_FTOI(vfp_to##name##p, fsz, sign, ) \
6618 CONV_FTOI(vfp_to##name##z##p, fsz, sign, _round_to_zero)
6619
6620 FLOAT_CONVS(si, s, 32, )
6621 FLOAT_CONVS(si, d, 64, )
6622 FLOAT_CONVS(ui, s, 32, u)
6623 FLOAT_CONVS(ui, d, 64, u)
6624
6625 #undef CONV_ITOF
6626 #undef CONV_FTOI
6627 #undef FLOAT_CONVS
6628
6629 /* floating point conversion */
6630 float64 VFP_HELPER(fcvtd, s)(float32 x, CPUARMState *env)
6631 {
6632     float64 r = float32_to_float64(x, &env->vfp.fp_status);
6633     /* ARM requires that S<->D conversion of any kind of NaN generates
6634      * a quiet NaN by forcing the most significant frac bit to 1.
6635      */
6636     return float64_maybe_silence_nan(r);
6637 }
6638
6639 float32 VFP_HELPER(fcvts, d)(float64 x, CPUARMState *env)
6640 {
6641     float32 r =  float64_to_float32(x, &env->vfp.fp_status);
6642     /* ARM requires that S<->D conversion of any kind of NaN generates
6643      * a quiet NaN by forcing the most significant frac bit to 1.
6644      */
6645     return float32_maybe_silence_nan(r);
6646 }
6647
6648 /* VFP3 fixed point conversion.  */
6649 #define VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype) \
6650 float##fsz HELPER(vfp_##name##to##p)(uint##isz##_t  x, uint32_t shift, \
6651                                      void *fpstp) \
6652 { \
6653     float_status *fpst = fpstp; \
6654     float##fsz tmp; \
6655     tmp = itype##_to_##float##fsz(x, fpst); \
6656     return float##fsz##_scalbn(tmp, -(int)shift, fpst); \
6657 }
6658
6659 /* Notice that we want only input-denormal exception flags from the
6660  * scalbn operation: the other possible flags (overflow+inexact if
6661  * we overflow to infinity, output-denormal) aren't correct for the
6662  * complete scale-and-convert operation.
6663  */
6664 #define VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, round) \
6665 uint##isz##_t HELPER(vfp_to##name##p##round)(float##fsz x, \
6666                                              uint32_t shift, \
6667                                              void *fpstp) \
6668 { \
6669     float_status *fpst = fpstp; \
6670     int old_exc_flags = get_float_exception_flags(fpst); \
6671     float##fsz tmp; \
6672     if (float##fsz##_is_any_nan(x)) { \
6673         float_raise(float_flag_invalid, fpst); \
6674         return 0; \
6675     } \
6676     tmp = float##fsz##_scalbn(x, shift, fpst); \
6677     old_exc_flags |= get_float_exception_flags(fpst) \
6678         & float_flag_input_denormal; \
6679     set_float_exception_flags(old_exc_flags, fpst); \
6680     return float##fsz##_to_##itype##round(tmp, fpst); \
6681 }
6682
6683 #define VFP_CONV_FIX(name, p, fsz, isz, itype)                   \
6684 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
6685 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, _round_to_zero) \
6686 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
6687
6688 #define VFP_CONV_FIX_A64(name, p, fsz, isz, itype)               \
6689 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
6690 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
6691
6692 VFP_CONV_FIX(sh, d, 64, 64, int16)
6693 VFP_CONV_FIX(sl, d, 64, 64, int32)
6694 VFP_CONV_FIX_A64(sq, d, 64, 64, int64)
6695 VFP_CONV_FIX(uh, d, 64, 64, uint16)
6696 VFP_CONV_FIX(ul, d, 64, 64, uint32)
6697 VFP_CONV_FIX_A64(uq, d, 64, 64, uint64)
6698 VFP_CONV_FIX(sh, s, 32, 32, int16)
6699 VFP_CONV_FIX(sl, s, 32, 32, int32)
6700 VFP_CONV_FIX_A64(sq, s, 32, 64, int64)
6701 VFP_CONV_FIX(uh, s, 32, 32, uint16)
6702 VFP_CONV_FIX(ul, s, 32, 32, uint32)
6703 VFP_CONV_FIX_A64(uq, s, 32, 64, uint64)
6704 #undef VFP_CONV_FIX
6705 #undef VFP_CONV_FIX_FLOAT
6706 #undef VFP_CONV_FLOAT_FIX_ROUND
6707
6708 /* Set the current fp rounding mode and return the old one.
6709  * The argument is a softfloat float_round_ value.
6710  */
6711 uint32_t HELPER(set_rmode)(uint32_t rmode, CPUARMState *env)
6712 {
6713     float_status *fp_status = &env->vfp.fp_status;
6714
6715     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
6716     set_float_rounding_mode(rmode, fp_status);
6717
6718     return prev_rmode;
6719 }
6720
6721 /* Set the current fp rounding mode in the standard fp status and return
6722  * the old one. This is for NEON instructions that need to change the
6723  * rounding mode but wish to use the standard FPSCR values for everything
6724  * else. Always set the rounding mode back to the correct value after
6725  * modifying it.
6726  * The argument is a softfloat float_round_ value.
6727  */
6728 uint32_t HELPER(set_neon_rmode)(uint32_t rmode, CPUARMState *env)
6729 {
6730     float_status *fp_status = &env->vfp.standard_fp_status;
6731
6732     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
6733     set_float_rounding_mode(rmode, fp_status);
6734
6735     return prev_rmode;
6736 }
6737
6738 /* Half precision conversions.  */
6739 static float32 do_fcvt_f16_to_f32(uint32_t a, CPUARMState *env, float_status *s)
6740 {
6741     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6742     float32 r = float16_to_float32(make_float16(a), ieee, s);
6743     if (ieee) {
6744         return float32_maybe_silence_nan(r);
6745     }
6746     return r;
6747 }
6748
6749 static uint32_t do_fcvt_f32_to_f16(float32 a, CPUARMState *env, float_status *s)
6750 {
6751     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6752     float16 r = float32_to_float16(a, ieee, s);
6753     if (ieee) {
6754         r = float16_maybe_silence_nan(r);
6755     }
6756     return float16_val(r);
6757 }
6758
6759 float32 HELPER(neon_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
6760 {
6761     return do_fcvt_f16_to_f32(a, env, &env->vfp.standard_fp_status);
6762 }
6763
6764 uint32_t HELPER(neon_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
6765 {
6766     return do_fcvt_f32_to_f16(a, env, &env->vfp.standard_fp_status);
6767 }
6768
6769 float32 HELPER(vfp_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
6770 {
6771     return do_fcvt_f16_to_f32(a, env, &env->vfp.fp_status);
6772 }
6773
6774 uint32_t HELPER(vfp_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
6775 {
6776     return do_fcvt_f32_to_f16(a, env, &env->vfp.fp_status);
6777 }
6778
6779 float64 HELPER(vfp_fcvt_f16_to_f64)(uint32_t a, CPUARMState *env)
6780 {
6781     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6782     float64 r = float16_to_float64(make_float16(a), ieee, &env->vfp.fp_status);
6783     if (ieee) {
6784         return float64_maybe_silence_nan(r);
6785     }
6786     return r;
6787 }
6788
6789 uint32_t HELPER(vfp_fcvt_f64_to_f16)(float64 a, CPUARMState *env)
6790 {
6791     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6792     float16 r = float64_to_float16(a, ieee, &env->vfp.fp_status);
6793     if (ieee) {
6794         r = float16_maybe_silence_nan(r);
6795     }
6796     return float16_val(r);
6797 }
6798
6799 #define float32_two make_float32(0x40000000)
6800 #define float32_three make_float32(0x40400000)
6801 #define float32_one_point_five make_float32(0x3fc00000)
6802
6803 float32 HELPER(recps_f32)(float32 a, float32 b, CPUARMState *env)
6804 {
6805     float_status *s = &env->vfp.standard_fp_status;
6806     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
6807         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
6808         if (!(float32_is_zero(a) || float32_is_zero(b))) {
6809             float_raise(float_flag_input_denormal, s);
6810         }
6811         return float32_two;
6812     }
6813     return float32_sub(float32_two, float32_mul(a, b, s), s);
6814 }
6815
6816 float32 HELPER(rsqrts_f32)(float32 a, float32 b, CPUARMState *env)
6817 {
6818     float_status *s = &env->vfp.standard_fp_status;
6819     float32 product;
6820     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
6821         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
6822         if (!(float32_is_zero(a) || float32_is_zero(b))) {
6823             float_raise(float_flag_input_denormal, s);
6824         }
6825         return float32_one_point_five;
6826     }
6827     product = float32_mul(a, b, s);
6828     return float32_div(float32_sub(float32_three, product, s), float32_two, s);
6829 }
6830
6831 /* NEON helpers.  */
6832
6833 /* Constants 256 and 512 are used in some helpers; we avoid relying on
6834  * int->float conversions at run-time.  */
6835 #define float64_256 make_float64(0x4070000000000000LL)
6836 #define float64_512 make_float64(0x4080000000000000LL)
6837 #define float32_maxnorm make_float32(0x7f7fffff)
6838 #define float64_maxnorm make_float64(0x7fefffffffffffffLL)
6839
6840 /* Reciprocal functions
6841  *
6842  * The algorithm that must be used to calculate the estimate
6843  * is specified by the ARM ARM, see FPRecipEstimate()
6844  */
6845
6846 static float64 recip_estimate(float64 a, float_status *real_fp_status)
6847 {
6848     /* These calculations mustn't set any fp exception flags,
6849      * so we use a local copy of the fp_status.
6850      */
6851     float_status dummy_status = *real_fp_status;
6852     float_status *s = &dummy_status;
6853     /* q = (int)(a * 512.0) */
6854     float64 q = float64_mul(float64_512, a, s);
6855     int64_t q_int = float64_to_int64_round_to_zero(q, s);
6856
6857     /* r = 1.0 / (((double)q + 0.5) / 512.0) */
6858     q = int64_to_float64(q_int, s);
6859     q = float64_add(q, float64_half, s);
6860     q = float64_div(q, float64_512, s);
6861     q = float64_div(float64_one, q, s);
6862
6863     /* s = (int)(256.0 * r + 0.5) */
6864     q = float64_mul(q, float64_256, s);
6865     q = float64_add(q, float64_half, s);
6866     q_int = float64_to_int64_round_to_zero(q, s);
6867
6868     /* return (double)s / 256.0 */
6869     return float64_div(int64_to_float64(q_int, s), float64_256, s);
6870 }
6871
6872 /* Common wrapper to call recip_estimate */
6873 static float64 call_recip_estimate(float64 num, int off, float_status *fpst)
6874 {
6875     uint64_t val64 = float64_val(num);
6876     uint64_t frac = extract64(val64, 0, 52);
6877     int64_t exp = extract64(val64, 52, 11);
6878     uint64_t sbit;
6879     float64 scaled, estimate;
6880
6881     /* Generate the scaled number for the estimate function */
6882     if (exp == 0) {
6883         if (extract64(frac, 51, 1) == 0) {
6884             exp = -1;
6885             frac = extract64(frac, 0, 50) << 2;
6886         } else {
6887             frac = extract64(frac, 0, 51) << 1;
6888         }
6889     }
6890
6891     /* scaled = '0' : '01111111110' : fraction<51:44> : Zeros(44); */
6892     scaled = make_float64((0x3feULL << 52)
6893                           | extract64(frac, 44, 8) << 44);
6894
6895     estimate = recip_estimate(scaled, fpst);
6896
6897     /* Build new result */
6898     val64 = float64_val(estimate);
6899     sbit = 0x8000000000000000ULL & val64;
6900     exp = off - exp;
6901     frac = extract64(val64, 0, 52);
6902
6903     if (exp == 0) {
6904         frac = 1ULL << 51 | extract64(frac, 1, 51);
6905     } else if (exp == -1) {
6906         frac = 1ULL << 50 | extract64(frac, 2, 50);
6907         exp = 0;
6908     }
6909
6910     return make_float64(sbit | (exp << 52) | frac);
6911 }
6912
6913 static bool round_to_inf(float_status *fpst, bool sign_bit)
6914 {
6915     switch (fpst->float_rounding_mode) {
6916     case float_round_nearest_even: /* Round to Nearest */
6917         return true;
6918     case float_round_up: /* Round to +Inf */
6919         return !sign_bit;
6920     case float_round_down: /* Round to -Inf */
6921         return sign_bit;
6922     case float_round_to_zero: /* Round to Zero */
6923         return false;
6924     }
6925
6926     g_assert_not_reached();
6927 }
6928
6929 float32 HELPER(recpe_f32)(float32 input, void *fpstp)
6930 {
6931     float_status *fpst = fpstp;
6932     float32 f32 = float32_squash_input_denormal(input, fpst);
6933     uint32_t f32_val = float32_val(f32);
6934     uint32_t f32_sbit = 0x80000000ULL & f32_val;
6935     int32_t f32_exp = extract32(f32_val, 23, 8);
6936     uint32_t f32_frac = extract32(f32_val, 0, 23);
6937     float64 f64, r64;
6938     uint64_t r64_val;
6939     int64_t r64_exp;
6940     uint64_t r64_frac;
6941
6942     if (float32_is_any_nan(f32)) {
6943         float32 nan = f32;
6944         if (float32_is_signaling_nan(f32)) {
6945             float_raise(float_flag_invalid, fpst);
6946             nan = float32_maybe_silence_nan(f32);
6947         }
6948         if (fpst->default_nan_mode) {
6949             nan =  float32_default_nan;
6950         }
6951         return nan;
6952     } else if (float32_is_infinity(f32)) {
6953         return float32_set_sign(float32_zero, float32_is_neg(f32));
6954     } else if (float32_is_zero(f32)) {
6955         float_raise(float_flag_divbyzero, fpst);
6956         return float32_set_sign(float32_infinity, float32_is_neg(f32));
6957     } else if ((f32_val & ~(1ULL << 31)) < (1ULL << 21)) {
6958         /* Abs(value) < 2.0^-128 */
6959         float_raise(float_flag_overflow | float_flag_inexact, fpst);
6960         if (round_to_inf(fpst, f32_sbit)) {
6961             return float32_set_sign(float32_infinity, float32_is_neg(f32));
6962         } else {
6963             return float32_set_sign(float32_maxnorm, float32_is_neg(f32));
6964         }
6965     } else if (f32_exp >= 253 && fpst->flush_to_zero) {
6966         float_raise(float_flag_underflow, fpst);
6967         return float32_set_sign(float32_zero, float32_is_neg(f32));
6968     }
6969
6970
6971     f64 = make_float64(((int64_t)(f32_exp) << 52) | (int64_t)(f32_frac) << 29);
6972     r64 = call_recip_estimate(f64, 253, fpst);
6973     r64_val = float64_val(r64);
6974     r64_exp = extract64(r64_val, 52, 11);
6975     r64_frac = extract64(r64_val, 0, 52);
6976
6977     /* result = sign : result_exp<7:0> : fraction<51:29>; */
6978     return make_float32(f32_sbit |
6979                         (r64_exp & 0xff) << 23 |
6980                         extract64(r64_frac, 29, 24));
6981 }
6982
6983 float64 HELPER(recpe_f64)(float64 input, void *fpstp)
6984 {
6985     float_status *fpst = fpstp;
6986     float64 f64 = float64_squash_input_denormal(input, fpst);
6987     uint64_t f64_val = float64_val(f64);
6988     uint64_t f64_sbit = 0x8000000000000000ULL & f64_val;
6989     int64_t f64_exp = extract64(f64_val, 52, 11);
6990     float64 r64;
6991     uint64_t r64_val;
6992     int64_t r64_exp;
6993     uint64_t r64_frac;
6994
6995     /* Deal with any special cases */
6996     if (float64_is_any_nan(f64)) {
6997         float64 nan = f64;
6998         if (float64_is_signaling_nan(f64)) {
6999             float_raise(float_flag_invalid, fpst);
7000             nan = float64_maybe_silence_nan(f64);
7001         }
7002         if (fpst->default_nan_mode) {
7003             nan =  float64_default_nan;
7004         }
7005         return nan;
7006     } else if (float64_is_infinity(f64)) {
7007         return float64_set_sign(float64_zero, float64_is_neg(f64));
7008     } else if (float64_is_zero(f64)) {
7009         float_raise(float_flag_divbyzero, fpst);
7010         return float64_set_sign(float64_infinity, float64_is_neg(f64));
7011     } else if ((f64_val & ~(1ULL << 63)) < (1ULL << 50)) {
7012         /* Abs(value) < 2.0^-1024 */
7013         float_raise(float_flag_overflow | float_flag_inexact, fpst);
7014         if (round_to_inf(fpst, f64_sbit)) {
7015             return float64_set_sign(float64_infinity, float64_is_neg(f64));
7016         } else {
7017             return float64_set_sign(float64_maxnorm, float64_is_neg(f64));
7018         }
7019     } else if (f64_exp >= 2045 && fpst->flush_to_zero) {
7020         float_raise(float_flag_underflow, fpst);
7021         return float64_set_sign(float64_zero, float64_is_neg(f64));
7022     }
7023
7024     r64 = call_recip_estimate(f64, 2045, fpst);
7025     r64_val = float64_val(r64);
7026     r64_exp = extract64(r64_val, 52, 11);
7027     r64_frac = extract64(r64_val, 0, 52);
7028
7029     /* result = sign : result_exp<10:0> : fraction<51:0> */
7030     return make_float64(f64_sbit |
7031                         ((r64_exp & 0x7ff) << 52) |
7032                         r64_frac);
7033 }
7034
7035 /* The algorithm that must be used to calculate the estimate
7036  * is specified by the ARM ARM.
7037  */
7038 static float64 recip_sqrt_estimate(float64 a, float_status *real_fp_status)
7039 {
7040     /* These calculations mustn't set any fp exception flags,
7041      * so we use a local copy of the fp_status.
7042      */
7043     float_status dummy_status = *real_fp_status;
7044     float_status *s = &dummy_status;
7045     float64 q;
7046     int64_t q_int;
7047
7048     if (float64_lt(a, float64_half, s)) {
7049         /* range 0.25 <= a < 0.5 */
7050
7051         /* a in units of 1/512 rounded down */
7052         /* q0 = (int)(a * 512.0);  */
7053         q = float64_mul(float64_512, a, s);
7054         q_int = float64_to_int64_round_to_zero(q, s);
7055
7056         /* reciprocal root r */
7057         /* r = 1.0 / sqrt(((double)q0 + 0.5) / 512.0);  */
7058         q = int64_to_float64(q_int, s);
7059         q = float64_add(q, float64_half, s);
7060         q = float64_div(q, float64_512, s);
7061         q = float64_sqrt(q, s);
7062         q = float64_div(float64_one, q, s);
7063     } else {
7064         /* range 0.5 <= a < 1.0 */
7065
7066         /* a in units of 1/256 rounded down */
7067         /* q1 = (int)(a * 256.0); */
7068         q = float64_mul(float64_256, a, s);
7069         int64_t q_int = float64_to_int64_round_to_zero(q, s);
7070
7071         /* reciprocal root r */
7072         /* r = 1.0 /sqrt(((double)q1 + 0.5) / 256); */
7073         q = int64_to_float64(q_int, s);
7074         q = float64_add(q, float64_half, s);
7075         q = float64_div(q, float64_256, s);
7076         q = float64_sqrt(q, s);
7077         q = float64_div(float64_one, q, s);
7078     }
7079     /* r in units of 1/256 rounded to nearest */
7080     /* s = (int)(256.0 * r + 0.5); */
7081
7082     q = float64_mul(q, float64_256,s );
7083     q = float64_add(q, float64_half, s);
7084     q_int = float64_to_int64_round_to_zero(q, s);
7085
7086     /* return (double)s / 256.0;*/
7087     return float64_div(int64_to_float64(q_int, s), float64_256, s);
7088 }
7089
7090 float32 HELPER(rsqrte_f32)(float32 input, void *fpstp)
7091 {
7092     float_status *s = fpstp;
7093     float32 f32 = float32_squash_input_denormal(input, s);
7094     uint32_t val = float32_val(f32);
7095     uint32_t f32_sbit = 0x80000000 & val;
7096     int32_t f32_exp = extract32(val, 23, 8);
7097     uint32_t f32_frac = extract32(val, 0, 23);
7098     uint64_t f64_frac;
7099     uint64_t val64;
7100     int result_exp;
7101     float64 f64;
7102
7103     if (float32_is_any_nan(f32)) {
7104         float32 nan = f32;
7105         if (float32_is_signaling_nan(f32)) {
7106             float_raise(float_flag_invalid, s);
7107             nan = float32_maybe_silence_nan(f32);
7108         }
7109         if (s->default_nan_mode) {
7110             nan =  float32_default_nan;
7111         }
7112         return nan;
7113     } else if (float32_is_zero(f32)) {
7114         float_raise(float_flag_divbyzero, s);
7115         return float32_set_sign(float32_infinity, float32_is_neg(f32));
7116     } else if (float32_is_neg(f32)) {
7117         float_raise(float_flag_invalid, s);
7118         return float32_default_nan;
7119     } else if (float32_is_infinity(f32)) {
7120         return float32_zero;
7121     }
7122
7123     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
7124      * preserving the parity of the exponent.  */
7125
7126     f64_frac = ((uint64_t) f32_frac) << 29;
7127     if (f32_exp == 0) {
7128         while (extract64(f64_frac, 51, 1) == 0) {
7129             f64_frac = f64_frac << 1;
7130             f32_exp = f32_exp-1;
7131         }
7132         f64_frac = extract64(f64_frac, 0, 51) << 1;
7133     }
7134
7135     if (extract64(f32_exp, 0, 1) == 0) {
7136         f64 = make_float64(((uint64_t) f32_sbit) << 32
7137                            | (0x3feULL << 52)
7138                            | f64_frac);
7139     } else {
7140         f64 = make_float64(((uint64_t) f32_sbit) << 32
7141                            | (0x3fdULL << 52)
7142                            | f64_frac);
7143     }
7144
7145     result_exp = (380 - f32_exp) / 2;
7146
7147     f64 = recip_sqrt_estimate(f64, s);
7148
7149     val64 = float64_val(f64);
7150
7151     val = ((result_exp & 0xff) << 23)
7152         | ((val64 >> 29)  & 0x7fffff);
7153     return make_float32(val);
7154 }
7155
7156 float64 HELPER(rsqrte_f64)(float64 input, void *fpstp)
7157 {
7158     float_status *s = fpstp;
7159     float64 f64 = float64_squash_input_denormal(input, s);
7160     uint64_t val = float64_val(f64);
7161     uint64_t f64_sbit = 0x8000000000000000ULL & val;
7162     int64_t f64_exp = extract64(val, 52, 11);
7163     uint64_t f64_frac = extract64(val, 0, 52);
7164     int64_t result_exp;
7165     uint64_t result_frac;
7166
7167     if (float64_is_any_nan(f64)) {
7168         float64 nan = f64;
7169         if (float64_is_signaling_nan(f64)) {
7170             float_raise(float_flag_invalid, s);
7171             nan = float64_maybe_silence_nan(f64);
7172         }
7173         if (s->default_nan_mode) {
7174             nan =  float64_default_nan;
7175         }
7176         return nan;
7177     } else if (float64_is_zero(f64)) {
7178         float_raise(float_flag_divbyzero, s);
7179         return float64_set_sign(float64_infinity, float64_is_neg(f64));
7180     } else if (float64_is_neg(f64)) {
7181         float_raise(float_flag_invalid, s);
7182         return float64_default_nan;
7183     } else if (float64_is_infinity(f64)) {
7184         return float64_zero;
7185     }
7186
7187     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
7188      * preserving the parity of the exponent.  */
7189
7190     if (f64_exp == 0) {
7191         while (extract64(f64_frac, 51, 1) == 0) {
7192             f64_frac = f64_frac << 1;
7193             f64_exp = f64_exp - 1;
7194         }
7195         f64_frac = extract64(f64_frac, 0, 51) << 1;
7196     }
7197
7198     if (extract64(f64_exp, 0, 1) == 0) {
7199         f64 = make_float64(f64_sbit
7200                            | (0x3feULL << 52)
7201                            | f64_frac);
7202     } else {
7203         f64 = make_float64(f64_sbit
7204                            | (0x3fdULL << 52)
7205                            | f64_frac);
7206     }
7207
7208     result_exp = (3068 - f64_exp) / 2;
7209
7210     f64 = recip_sqrt_estimate(f64, s);
7211
7212     result_frac = extract64(float64_val(f64), 0, 52);
7213
7214     return make_float64(f64_sbit |
7215                         ((result_exp & 0x7ff) << 52) |
7216                         result_frac);
7217 }
7218
7219 uint32_t HELPER(recpe_u32)(uint32_t a, void *fpstp)
7220 {
7221     float_status *s = fpstp;
7222     float64 f64;
7223
7224     if ((a & 0x80000000) == 0) {
7225         return 0xffffffff;
7226     }
7227
7228     f64 = make_float64((0x3feULL << 52)
7229                        | ((int64_t)(a & 0x7fffffff) << 21));
7230
7231     f64 = recip_estimate(f64, s);
7232
7233     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
7234 }
7235
7236 uint32_t HELPER(rsqrte_u32)(uint32_t a, void *fpstp)
7237 {
7238     float_status *fpst = fpstp;
7239     float64 f64;
7240
7241     if ((a & 0xc0000000) == 0) {
7242         return 0xffffffff;
7243     }
7244
7245     if (a & 0x80000000) {
7246         f64 = make_float64((0x3feULL << 52)
7247                            | ((uint64_t)(a & 0x7fffffff) << 21));
7248     } else { /* bits 31-30 == '01' */
7249         f64 = make_float64((0x3fdULL << 52)
7250                            | ((uint64_t)(a & 0x3fffffff) << 22));
7251     }
7252
7253     f64 = recip_sqrt_estimate(f64, fpst);
7254
7255     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
7256 }
7257
7258 /* VFPv4 fused multiply-accumulate */
7259 float32 VFP_HELPER(muladd, s)(float32 a, float32 b, float32 c, void *fpstp)
7260 {
7261     float_status *fpst = fpstp;
7262     return float32_muladd(a, b, c, 0, fpst);
7263 }
7264
7265 float64 VFP_HELPER(muladd, d)(float64 a, float64 b, float64 c, void *fpstp)
7266 {
7267     float_status *fpst = fpstp;
7268     return float64_muladd(a, b, c, 0, fpst);
7269 }
7270
7271 /* ARMv8 round to integral */
7272 float32 HELPER(rints_exact)(float32 x, void *fp_status)
7273 {
7274     return float32_round_to_int(x, fp_status);
7275 }
7276
7277 float64 HELPER(rintd_exact)(float64 x, void *fp_status)
7278 {
7279     return float64_round_to_int(x, fp_status);
7280 }
7281
7282 float32 HELPER(rints)(float32 x, void *fp_status)
7283 {
7284     int old_flags = get_float_exception_flags(fp_status), new_flags;
7285     float32 ret;
7286
7287     ret = float32_round_to_int(x, fp_status);
7288
7289     /* Suppress any inexact exceptions the conversion produced */
7290     if (!(old_flags & float_flag_inexact)) {
7291         new_flags = get_float_exception_flags(fp_status);
7292         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
7293     }
7294
7295     return ret;
7296 }
7297
7298 float64 HELPER(rintd)(float64 x, void *fp_status)
7299 {
7300     int old_flags = get_float_exception_flags(fp_status), new_flags;
7301     float64 ret;
7302
7303     ret = float64_round_to_int(x, fp_status);
7304
7305     new_flags = get_float_exception_flags(fp_status);
7306
7307     /* Suppress any inexact exceptions the conversion produced */
7308     if (!(old_flags & float_flag_inexact)) {
7309         new_flags = get_float_exception_flags(fp_status);
7310         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
7311     }
7312
7313     return ret;
7314 }
7315
7316 /* Convert ARM rounding mode to softfloat */
7317 int arm_rmode_to_sf(int rmode)
7318 {
7319     switch (rmode) {
7320     case FPROUNDING_TIEAWAY:
7321         rmode = float_round_ties_away;
7322         break;
7323     case FPROUNDING_ODD:
7324         /* FIXME: add support for TIEAWAY and ODD */
7325         qemu_log_mask(LOG_UNIMP, "arm: unimplemented rounding mode: %d\n",
7326                       rmode);
7327     case FPROUNDING_TIEEVEN:
7328     default:
7329         rmode = float_round_nearest_even;
7330         break;
7331     case FPROUNDING_POSINF:
7332         rmode = float_round_up;
7333         break;
7334     case FPROUNDING_NEGINF:
7335         rmode = float_round_down;
7336         break;
7337     case FPROUNDING_ZERO:
7338         rmode = float_round_to_zero;
7339         break;
7340     }
7341     return rmode;
7342 }
7343
7344 /* CRC helpers.
7345  * The upper bytes of val (above the number specified by 'bytes') must have
7346  * been zeroed out by the caller.
7347  */
7348 uint32_t HELPER(crc32)(uint32_t acc, uint32_t val, uint32_t bytes)
7349 {
7350     uint8_t buf[4];
7351
7352     stl_le_p(buf, val);
7353
7354     /* zlib crc32 converts the accumulator and output to one's complement.  */
7355     return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
7356 }
7357
7358 uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
7359 {
7360     uint8_t buf[4];
7361
7362     stl_le_p(buf, val);
7363
7364     /* Linux crc32c converts the output to one's complement.  */
7365     return crc32c(acc, buf, bytes) ^ 0xffffffff;
7366 }