target-arm/helper.c

   1 #include "cpu.h"
   2 #include "internals.h"
   3 #include "exec/gdbstub.h"
   4 #include "exec/helper-proto.h"
   5 #include "qemu/host-utils.h"
   6 #include "sysemu/arch_init.h"
   7 #include "sysemu/sysemu.h"
   8 #include "qemu/bitops.h"
   9 #include "qemu/crc32c.h"
  10 #include "exec/cpu_ldst.h"
  11 #include "arm_ldst.h"
  12 #include <zlib.h> /* For crc32 */
  13
  14 #ifndef CONFIG_USER_ONLY
  15 static inline int get_phys_addr(CPUARMState *env, target_ulong address,
  16                                 int access_type, ARMMMUIdx mmu_idx,
  17                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
  18                                 target_ulong *page_size);
  19
  20 /* Definitions for the PMCCNTR and PMCR registers */
  21 #define PMCRD   0x8
  22 #define PMCRC   0x4
  23 #define PMCRE   0x1
  24 #endif
  25
  26 static int vfp_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  27 {
  28     int nregs;
  29
  30     /* VFP data registers are always little-endian.  */
  31     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  32     if (reg < nregs) {
  33         stfq_le_p(buf, env->vfp.regs[reg]);
  34         return 8;
  35     }
  36     if (arm_feature(env, ARM_FEATURE_NEON)) {
  37         /* Aliases for Q regs.  */
  38         nregs += 16;
  39         if (reg < nregs) {
  40             stfq_le_p(buf, env->vfp.regs[(reg - 32) * 2]);
  41             stfq_le_p(buf + 8, env->vfp.regs[(reg - 32) * 2 + 1]);
  42             return 16;
  43         }
  44     }
  45     switch (reg - nregs) {
  46     case 0: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSID]); return 4;
  47     case 1: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSCR]); return 4;
  48     case 2: stl_p(buf, env->vfp.xregs[ARM_VFP_FPEXC]); return 4;
  49     }
  50     return 0;
  51 }
  52
  53 static int vfp_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
  54 {
  55     int nregs;
  56
  57     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  58     if (reg < nregs) {
  59         env->vfp.regs[reg] = ldfq_le_p(buf);
  60         return 8;
  61     }
  62     if (arm_feature(env, ARM_FEATURE_NEON)) {
  63         nregs += 16;
  64         if (reg < nregs) {
  65             env->vfp.regs[(reg - 32) * 2] = ldfq_le_p(buf);
  66             env->vfp.regs[(reg - 32) * 2 + 1] = ldfq_le_p(buf + 8);
  67             return 16;
  68         }
  69     }
  70     switch (reg - nregs) {
  71     case 0: env->vfp.xregs[ARM_VFP_FPSID] = ldl_p(buf); return 4;
  72     case 1: env->vfp.xregs[ARM_VFP_FPSCR] = ldl_p(buf); return 4;
  73     case 2: env->vfp.xregs[ARM_VFP_FPEXC] = ldl_p(buf) & (1 << 30); return 4;
  74     }
  75     return 0;
  76 }
  77
  78 static int aarch64_fpu_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  79 {
  80     switch (reg) {
  81     case 0 ... 31:
  82         /* 128 bit FP register */
  83         stfq_le_p(buf, env->vfp.regs[reg * 2]);
  84         stfq_le_p(buf + 8, env->vfp.regs[reg * 2 + 1]);
  85         return 16;
  86     case 32:
  87         /* FPSR */
  88         stl_p(buf, vfp_get_fpsr(env));
  89         return 4;
  90     case 33:
  91         /* FPCR */
  92         stl_p(buf, vfp_get_fpcr(env));
  93         return 4;
  94     default:
  95         return 0;
  96     }
  97 }
  98
  99 static int aarch64_fpu_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
 100 {
 101     switch (reg) {
 102     case 0 ... 31:
 103         /* 128 bit FP register */
 104         env->vfp.regs[reg * 2] = ldfq_le_p(buf);
 105         env->vfp.regs[reg * 2 + 1] = ldfq_le_p(buf + 8);
 106         return 16;
 107     case 32:
 108         /* FPSR */
 109         vfp_set_fpsr(env, ldl_p(buf));
 110         return 4;
 111     case 33:
 112         /* FPCR */
 113         vfp_set_fpcr(env, ldl_p(buf));
 114         return 4;
 115     default:
 116         return 0;
 117     }
 118 }
 119
 120 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
 121 {
 122     assert(ri->fieldoffset);
 123     if (cpreg_field_is_64bit(ri)) {
 124         return CPREG_FIELD64(env, ri);
 125     } else {
 126         return CPREG_FIELD32(env, ri);
 127     }
 128 }
 129
 130 static void raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
 131                       uint64_t value)
 132 {
 133     assert(ri->fieldoffset);
 134     if (cpreg_field_is_64bit(ri)) {
 135         CPREG_FIELD64(env, ri) = value;
 136     } else {
 137         CPREG_FIELD32(env, ri) = value;
 138     }
 139 }
 140
 141 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 142 {
 143     return (char *)env + ri->fieldoffset;
 144 }
 145
 146 static uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 147 {
 148     /* Raw read of a coprocessor register (as needed for migration, etc). */
 149     if (ri->type & ARM_CP_CONST) {
 150         return ri->resetvalue;
 151     } else if (ri->raw_readfn) {
 152         return ri->raw_readfn(env, ri);
 153     } else if (ri->readfn) {
 154         return ri->readfn(env, ri);
 155     } else {
 156         return raw_read(env, ri);
 157     }
 158 }
 159
 160 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
 161                              uint64_t v)
 162 {
 163     /* Raw write of a coprocessor register (as needed for migration, etc).
 164      * Note that constant registers are treated as write-ignored; the
 165      * caller should check for success by whether a readback gives the
 166      * value written.
 167      */
 168     if (ri->type & ARM_CP_CONST) {
 169         return;
 170     } else if (ri->raw_writefn) {
 171         ri->raw_writefn(env, ri, v);
 172     } else if (ri->writefn) {
 173         ri->writefn(env, ri, v);
 174     } else {
 175         raw_write(env, ri, v);
 176     }
 177 }
 178
 179 static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
 180 {
 181    /* Return true if the regdef would cause an assertion if you called
 182     * read_raw_cp_reg() or write_raw_cp_reg() on it (ie if it is a
 183     * program bug for it not to have the NO_RAW flag).
 184     * NB that returning false here doesn't necessarily mean that calling
 185     * read/write_raw_cp_reg() is safe, because we can't distinguish "has
 186     * read/write access functions which are safe for raw use" from "has
 187     * read/write access functions which have side effects but has forgotten
 188     * to provide raw access functions".
 189     * The tests here line up with the conditions in read/write_raw_cp_reg()
 190     * and assertions in raw_read()/raw_write().
 191     */
 192     if ((ri->type & ARM_CP_CONST) ||
 193         ri->fieldoffset ||
 194         ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
 195         return false;
 196     }
 197     return true;
 198 }
 199
 200 bool write_cpustate_to_list(ARMCPU *cpu)
 201 {
 202     /* Write the coprocessor state from cpu->env to the (index,value) list. */
 203     int i;
 204     bool ok = true;
 205
 206     for (i = 0; i < cpu->cpreg_array_len; i++) {
 207         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 208         const ARMCPRegInfo *ri;
 209
 210         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 211         if (!ri) {
 212             ok = false;
 213             continue;
 214         }
 215         if (ri->type & ARM_CP_NO_RAW) {
 216             continue;
 217         }
 218         cpu->cpreg_values[i] = read_raw_cp_reg(&cpu->env, ri);
 219     }
 220     return ok;
 221 }
 222
 223 bool write_list_to_cpustate(ARMCPU *cpu)
 224 {
 225     int i;
 226     bool ok = true;
 227
 228     for (i = 0; i < cpu->cpreg_array_len; i++) {
 229         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 230         uint64_t v = cpu->cpreg_values[i];
 231         const ARMCPRegInfo *ri;
 232
 233         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 234         if (!ri) {
 235             ok = false;
 236             continue;
 237         }
 238         if (ri->type & ARM_CP_NO_RAW) {
 239             continue;
 240         }
 241         /* Write value and confirm it reads back as written
 242          * (to catch read-only registers and partially read-only
 243          * registers where the incoming migration value doesn't match)
 244          */
 245         write_raw_cp_reg(&cpu->env, ri, v);
 246         if (read_raw_cp_reg(&cpu->env, ri) != v) {
 247             ok = false;
 248         }
 249     }
 250     return ok;
 251 }
 252
 253 static void add_cpreg_to_list(gpointer key, gpointer opaque)
 254 {
 255     ARMCPU *cpu = opaque;
 256     uint64_t regidx;
 257     const ARMCPRegInfo *ri;
 258
 259     regidx = *(uint32_t *)key;
 260     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 261
 262     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 263         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
 264         /* The value array need not be initialized at this point */
 265         cpu->cpreg_array_len++;
 266     }
 267 }
 268
 269 static void count_cpreg(gpointer key, gpointer opaque)
 270 {
 271     ARMCPU *cpu = opaque;
 272     uint64_t regidx;
 273     const ARMCPRegInfo *ri;
 274
 275     regidx = *(uint32_t *)key;
 276     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 277
 278     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 279         cpu->cpreg_array_len++;
 280     }
 281 }
 282
 283 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
 284 {
 285     uint64_t aidx = cpreg_to_kvm_id(*(uint32_t *)a);
 286     uint64_t bidx = cpreg_to_kvm_id(*(uint32_t *)b);
 287
 288     if (aidx > bidx) {
 289         return 1;
 290     }
 291     if (aidx < bidx) {
 292         return -1;
 293     }
 294     return 0;
 295 }
 296
 297 static void cpreg_make_keylist(gpointer key, gpointer value, gpointer udata)
 298 {
 299     GList **plist = udata;
 300
 301     *plist = g_list_prepend(*plist, key);
 302 }
 303
 304 void init_cpreg_list(ARMCPU *cpu)
 305 {
 306     /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
 307      * Note that we require cpreg_tuples[] to be sorted by key ID.
 308      */
 309     GList *keys = NULL;
 310     int arraylen;
 311
 312     g_hash_table_foreach(cpu->cp_regs, cpreg_make_keylist, &keys);
 313
 314     keys = g_list_sort(keys, cpreg_key_compare);
 315
 316     cpu->cpreg_array_len = 0;
 317
 318     g_list_foreach(keys, count_cpreg, cpu);
 319
 320     arraylen = cpu->cpreg_array_len;
 321     cpu->cpreg_indexes = g_new(uint64_t, arraylen);
 322     cpu->cpreg_values = g_new(uint64_t, arraylen);
 323     cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
 324     cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
 325     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
 326     cpu->cpreg_array_len = 0;
 327
 328     g_list_foreach(keys, add_cpreg_to_list, cpu);
 329
 330     assert(cpu->cpreg_array_len == arraylen);
 331
 332     g_list_free(keys);
 333 }
 334
 335 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 336 {
 337     ARMCPU *cpu = arm_env_get_cpu(env);
 338
 339     raw_write(env, ri, value);
 340     tlb_flush(CPU(cpu), 1); /* Flush TLB as domain not tracked in TLB */
 341 }
 342
 343 static void fcse_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 344 {
 345     ARMCPU *cpu = arm_env_get_cpu(env);
 346
 347     if (raw_read(env, ri) != value) {
 348         /* Unlike real hardware the qemu TLB uses virtual addresses,
 349          * not modified virtual addresses, so this causes a TLB flush.
 350          */
 351         tlb_flush(CPU(cpu), 1);
 352         raw_write(env, ri, value);
 353     }
 354 }
 355
 356 static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 357                              uint64_t value)
 358 {
 359     ARMCPU *cpu = arm_env_get_cpu(env);
 360
 361     if (raw_read(env, ri) != value && !arm_feature(env, ARM_FEATURE_MPU)
 362         && !extended_addresses_enabled(env)) {
 363         /* For VMSA (when not using the LPAE long descriptor page table
 364          * format) this register includes the ASID, so do a TLB flush.
 365          * For PMSA it is purely a process ID and no action is needed.
 366          */
 367         tlb_flush(CPU(cpu), 1);
 368     }
 369     raw_write(env, ri, value);
 370 }
 371
 372 static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
 373                           uint64_t value)
 374 {
 375     /* Invalidate all (TLBIALL) */
 376     ARMCPU *cpu = arm_env_get_cpu(env);
 377
 378     tlb_flush(CPU(cpu), 1);
 379 }
 380
 381 static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri,
 382                           uint64_t value)
 383 {
 384     /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */
 385     ARMCPU *cpu = arm_env_get_cpu(env);
 386
 387     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 388 }
 389
 390 static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri,
 391                            uint64_t value)
 392 {
 393     /* Invalidate by ASID (TLBIASID) */
 394     ARMCPU *cpu = arm_env_get_cpu(env);
 395
 396     tlb_flush(CPU(cpu), value == 0);
 397 }
 398
 399 static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
 400                            uint64_t value)
 401 {
 402     /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */
 403     ARMCPU *cpu = arm_env_get_cpu(env);
 404
 405     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 406 }
 407
 408 /* IS variants of TLB operations must affect all cores */
 409 static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 410                              uint64_t value)
 411 {
 412     CPUState *other_cs;
 413
 414     CPU_FOREACH(other_cs) {
 415         tlb_flush(other_cs, 1);
 416     }
 417 }
 418
 419 static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 420                              uint64_t value)
 421 {
 422     CPUState *other_cs;
 423
 424     CPU_FOREACH(other_cs) {
 425         tlb_flush(other_cs, value == 0);
 426     }
 427 }
 428
 429 static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 430                              uint64_t value)
 431 {
 432     CPUState *other_cs;
 433
 434     CPU_FOREACH(other_cs) {
 435         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 436     }
 437 }
 438
 439 static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 440                              uint64_t value)
 441 {
 442     CPUState *other_cs;
 443
 444     CPU_FOREACH(other_cs) {
 445         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 446     }
 447 }
 448
 449 static const ARMCPRegInfo cp_reginfo[] = {
 450     /* Define the secure and non-secure FCSE identifier CP registers
 451      * separately because there is no secure bank in V8 (no _EL3).  This allows
 452      * the secure register to be properly reset and migrated. There is also no
 453      * v8 EL1 version of the register so the non-secure instance stands alone.
 454      */
 455     { .name = "FCSEIDR(NS)",
 456       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 457       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 458       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
 459       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 460     { .name = "FCSEIDR(S)",
 461       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 462       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 463       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
 464       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 465     /* Define the secure and non-secure context identifier CP registers
 466      * separately because there is no secure bank in V8 (no _EL3).  This allows
 467      * the secure register to be properly reset and migrated.  In the
 468      * non-secure case, the 32-bit register will have reset and migration
 469      * disabled during registration as it is handled by the 64-bit instance.
 470      */
 471     { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
 472       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 473       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 474       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
 475       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 476     { .name = "CONTEXTIDR(S)", .state = ARM_CP_STATE_AA32,
 477       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 478       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 479       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
 480       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 481     REGINFO_SENTINEL
 482 };
 483
 484 static const ARMCPRegInfo not_v8_cp_reginfo[] = {
 485     /* NB: Some of these registers exist in v8 but with more precise
 486      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
 487      */
 488     /* MMU Domain access control / MPU write buffer control */
 489     { .name = "DACR",
 490       .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
 491       .access = PL1_RW, .resetvalue = 0,
 492       .writefn = dacr_write, .raw_writefn = raw_write,
 493       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
 494                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
 495     /* ??? This covers not just the impdef TLB lockdown registers but also
 496      * some v7VMSA registers relating to TEX remap, so it is overly broad.
 497      */
 498     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = CP_ANY,
 499       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 500     /* Cache maintenance ops; some of this space may be overridden later. */
 501     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
 502       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
 503       .type = ARM_CP_NOP | ARM_CP_OVERRIDE },
 504     REGINFO_SENTINEL
 505 };
 506
 507 static const ARMCPRegInfo not_v6_cp_reginfo[] = {
 508     /* Not all pre-v6 cores implemented this WFI, so this is slightly
 509      * over-broad.
 510      */
 511     { .name = "WFI_v5", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = 2,
 512       .access = PL1_W, .type = ARM_CP_WFI },
 513     REGINFO_SENTINEL
 514 };
 515
 516 static const ARMCPRegInfo not_v7_cp_reginfo[] = {
 517     /* Standard v6 WFI (also used in some pre-v6 cores); not in v7 (which
 518      * is UNPREDICTABLE; we choose to NOP as most implementations do).
 519      */
 520     { .name = "WFI_v6", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 521       .access = PL1_W, .type = ARM_CP_WFI },
 522     /* L1 cache lockdown. Not architectural in v6 and earlier but in practice
 523      * implemented in 926, 946, 1026, 1136, 1176 and 11MPCore. StrongARM and
 524      * OMAPCP will override this space.
 525      */
 526     { .name = "DLOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 0,
 527       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_data),
 528       .resetvalue = 0 },
 529     { .name = "ILOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 1,
 530       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_insn),
 531       .resetvalue = 0 },
 532     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
 533     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
 534       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
 535       .resetvalue = 0 },
 536     /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
 537      * implementing it as RAZ means the "debug architecture version" bits
 538      * will read as a reserved value, which should cause Linux to not try
 539      * to use the debug hardware.
 540      */
 541     { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
 542       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
 543     /* MMU TLB control. Note that the wildcarding means we cover not just
 544      * the unified TLB ops but also the dside/iside/inner-shareable variants.
 545      */
 546     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
 547       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
 548       .type = ARM_CP_NO_RAW },
 549     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
 550       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
 551       .type = ARM_CP_NO_RAW },
 552     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
 553       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
 554       .type = ARM_CP_NO_RAW },
 555     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
 556       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
 557       .type = ARM_CP_NO_RAW },
 558     REGINFO_SENTINEL
 559 };
 560
 561 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 562                         uint64_t value)
 563 {
 564     uint32_t mask = 0;
 565
 566     /* In ARMv8 most bits of CPACR_EL1 are RES0. */
 567     if (!arm_feature(env, ARM_FEATURE_V8)) {
 568         /* ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
 569          * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
 570          * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
 571          */
 572         if (arm_feature(env, ARM_FEATURE_VFP)) {
 573             /* VFP coprocessor: cp10 & cp11 [23:20] */
 574             mask |= (1 << 31) | (1 << 30) | (0xf << 20);
 575
 576             if (!arm_feature(env, ARM_FEATURE_NEON)) {
 577                 /* ASEDIS [31] bit is RAO/WI */
 578                 value |= (1 << 31);
 579             }
 580
 581             /* VFPv3 and upwards with NEON implement 32 double precision
 582              * registers (D0-D31).
 583              */
 584             if (!arm_feature(env, ARM_FEATURE_NEON) ||
 585                     !arm_feature(env, ARM_FEATURE_VFP3)) {
 586                 /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
 587                 value |= (1 << 30);
 588             }
 589         }
 590         value &= mask;
 591     }
 592     env->cp15.cpacr_el1 = value;
 593 }
 594
 595 static const ARMCPRegInfo v6_cp_reginfo[] = {
 596     /* prefetch by MVA in v6, NOP in v7 */
 597     { .name = "MVA_prefetch",
 598       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
 599       .access = PL1_W, .type = ARM_CP_NOP },
 600     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
 601       .access = PL0_W, .type = ARM_CP_NOP },
 602     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
 603       .access = PL0_W, .type = ARM_CP_NOP },
 604     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
 605       .access = PL0_W, .type = ARM_CP_NOP },
 606     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
 607       .access = PL1_RW,
 608       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
 609                              offsetof(CPUARMState, cp15.ifar_ns) },
 610       .resetvalue = 0, },
 611     /* Watchpoint Fault Address Register : should actually only be present
 612      * for 1136, 1176, 11MPCore.
 613      */
 614     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
 615       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
 616     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
 617       .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2,
 618       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.cpacr_el1),
 619       .resetvalue = 0, .writefn = cpacr_write },
 620     REGINFO_SENTINEL
 621 };
 622
 623 static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri)
 624 {
 625     /* Performance monitor registers user accessibility is controlled
 626      * by PMUSERENR.
 627      */
 628     if (arm_current_el(env) == 0 && !env->cp15.c9_pmuserenr) {
 629         return CP_ACCESS_TRAP;
 630     }
 631     return CP_ACCESS_OK;
 632 }
 633
 634 #ifndef CONFIG_USER_ONLY
 635
 636 static inline bool arm_ccnt_enabled(CPUARMState *env)
 637 {
 638     /* This does not support checking PMCCFILTR_EL0 register */
 639
 640     if (!(env->cp15.c9_pmcr & PMCRE)) {
 641         return false;
 642     }
 643
 644     return true;
 645 }
 646
 647 void pmccntr_sync(CPUARMState *env)
 648 {
 649     uint64_t temp_ticks;
 650
 651     temp_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 652                           get_ticks_per_sec(), 1000000);
 653
 654     if (env->cp15.c9_pmcr & PMCRD) {
 655         /* Increment once every 64 processor clock cycles */
 656         temp_ticks /= 64;
 657     }
 658
 659     if (arm_ccnt_enabled(env)) {
 660         env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
 661     }
 662 }
 663
 664 static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 665                        uint64_t value)
 666 {
 667     pmccntr_sync(env);
 668
 669     if (value & PMCRC) {
 670         /* The counter has been reset */
 671         env->cp15.c15_ccnt = 0;
 672     }
 673
 674     /* only the DP, X, D and E bits are writable */
 675     env->cp15.c9_pmcr &= ~0x39;
 676     env->cp15.c9_pmcr |= (value & 0x39);
 677
 678     pmccntr_sync(env);
 679 }
 680
 681 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 682 {
 683     uint64_t total_ticks;
 684
 685     if (!arm_ccnt_enabled(env)) {
 686         /* Counter is disabled, do not change value */
 687         return env->cp15.c15_ccnt;
 688     }
 689
 690     total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 691                            get_ticks_per_sec(), 1000000);
 692
 693     if (env->cp15.c9_pmcr & PMCRD) {
 694         /* Increment once every 64 processor clock cycles */
 695         total_ticks /= 64;
 696     }
 697     return total_ticks - env->cp15.c15_ccnt;
 698 }
 699
 700 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 701                         uint64_t value)
 702 {
 703     uint64_t total_ticks;
 704
 705     if (!arm_ccnt_enabled(env)) {
 706         /* Counter is disabled, set the absolute value */
 707         env->cp15.c15_ccnt = value;
 708         return;
 709     }
 710
 711     total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 712                            get_ticks_per_sec(), 1000000);
 713
 714     if (env->cp15.c9_pmcr & PMCRD) {
 715         /* Increment once every 64 processor clock cycles */
 716         total_ticks /= 64;
 717     }
 718     env->cp15.c15_ccnt = total_ticks - value;
 719 }
 720
 721 static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
 722                             uint64_t value)
 723 {
 724     uint64_t cur_val = pmccntr_read(env, NULL);
 725
 726     pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
 727 }
 728
 729 #else /* CONFIG_USER_ONLY */
 730
 731 void pmccntr_sync(CPUARMState *env)
 732 {
 733 }
 734
 735 #endif
 736
 737 static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 738                             uint64_t value)
 739 {
 740     pmccntr_sync(env);
 741     env->cp15.pmccfiltr_el0 = value & 0x7E000000;
 742     pmccntr_sync(env);
 743 }
 744
 745 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 746                             uint64_t value)
 747 {
 748     value &= (1 << 31);
 749     env->cp15.c9_pmcnten |= value;
 750 }
 751
 752 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 753                              uint64_t value)
 754 {
 755     value &= (1 << 31);
 756     env->cp15.c9_pmcnten &= ~value;
 757 }
 758
 759 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 760                          uint64_t value)
 761 {
 762     env->cp15.c9_pmovsr &= ~value;
 763 }
 764
 765 static void pmxevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
 766                              uint64_t value)
 767 {
 768     env->cp15.c9_pmxevtyper = value & 0xff;
 769 }
 770
 771 static void pmuserenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 772                             uint64_t value)
 773 {
 774     env->cp15.c9_pmuserenr = value & 1;
 775 }
 776
 777 static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 778                              uint64_t value)
 779 {
 780     /* We have no event counters so only the C bit can be changed */
 781     value &= (1 << 31);
 782     env->cp15.c9_pminten |= value;
 783 }
 784
 785 static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 786                              uint64_t value)
 787 {
 788     value &= (1 << 31);
 789     env->cp15.c9_pminten &= ~value;
 790 }
 791
 792 static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
 793                        uint64_t value)
 794 {
 795     /* Note that even though the AArch64 view of this register has bits
 796      * [10:0] all RES0 we can only mask the bottom 5, to comply with the
 797      * architectural requirements for bits which are RES0 only in some
 798      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
 799      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
 800      */
 801     raw_write(env, ri, value & ~0x1FULL);
 802 }
 803
 804 static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 805 {
 806     /* We only mask off bits that are RES0 both for AArch64 and AArch32.
 807      * For bits that vary between AArch32/64, code needs to check the
 808      * current execution mode before directly using the feature bit.
 809      */
 810     uint32_t valid_mask = SCR_AARCH64_MASK | SCR_AARCH32_MASK;
 811
 812     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 813         valid_mask &= ~SCR_HCE;
 814
 815         /* On ARMv7, SMD (or SCD as it is called in v7) is only
 816          * supported if EL2 exists. The bit is UNK/SBZP when
 817          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
 818          * when EL2 is unavailable.
 819          * On ARMv8, this bit is always available.
 820          */
 821         if (arm_feature(env, ARM_FEATURE_V7) &&
 822             !arm_feature(env, ARM_FEATURE_V8)) {
 823             valid_mask &= ~SCR_SMD;
 824         }
 825     }
 826
 827     /* Clear all-context RES0 bits.  */
 828     value &= valid_mask;
 829     raw_write(env, ri, value);
 830 }
 831
 832 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 833 {
 834     ARMCPU *cpu = arm_env_get_cpu(env);
 835
 836     /* Acquire the CSSELR index from the bank corresponding to the CCSIDR
 837      * bank
 838      */
 839     uint32_t index = A32_BANKED_REG_GET(env, csselr,
 840                                         ri->secure & ARM_CP_SECSTATE_S);
 841
 842     return cpu->ccsidr[index];
 843 }
 844
 845 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 846                          uint64_t value)
 847 {
 848     raw_write(env, ri, value & 0xf);
 849 }
 850
 851 static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 852 {
 853     CPUState *cs = ENV_GET_CPU(env);
 854     uint64_t ret = 0;
 855
 856     if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
 857         ret |= CPSR_I;
 858     }
 859     if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
 860         ret |= CPSR_F;
 861     }
 862     /* External aborts are not possible in QEMU so A bit is always clear */
 863     return ret;
 864 }
 865
 866 static const ARMCPRegInfo v7_cp_reginfo[] = {
 867     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
 868     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 869       .access = PL1_W, .type = ARM_CP_NOP },
 870     /* Performance monitors are implementation defined in v7,
 871      * but with an ARM recommended set of registers, which we
 872      * follow (although we don't actually implement any counters)
 873      *
 874      * Performance registers fall into three categories:
 875      *  (a) always UNDEF in PL0, RW in PL1 (PMINTENSET, PMINTENCLR)
 876      *  (b) RO in PL0 (ie UNDEF on write), RW in PL1 (PMUSERENR)
 877      *  (c) UNDEF in PL0 if PMUSERENR.EN==0, otherwise accessible (all others)
 878      * For the cases controlled by PMUSERENR we must set .access to PL0_RW
 879      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
 880      */
 881     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
 882       .access = PL0_RW, .type = ARM_CP_ALIAS,
 883       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
 884       .writefn = pmcntenset_write,
 885       .accessfn = pmreg_access,
 886       .raw_writefn = raw_write },
 887     { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64,
 888       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
 889       .access = PL0_RW, .accessfn = pmreg_access,
 890       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
 891       .writefn = pmcntenset_write, .raw_writefn = raw_write },
 892     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
 893       .access = PL0_RW,
 894       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
 895       .accessfn = pmreg_access,
 896       .writefn = pmcntenclr_write,
 897       .type = ARM_CP_ALIAS },
 898     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
 899       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
 900       .access = PL0_RW, .accessfn = pmreg_access,
 901       .type = ARM_CP_ALIAS,
 902       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
 903       .writefn = pmcntenclr_write },
 904     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
 905       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
 906       .accessfn = pmreg_access,
 907       .writefn = pmovsr_write,
 908       .raw_writefn = raw_write },
 909     /* Unimplemented so WI. */
 910     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
 911       .access = PL0_W, .accessfn = pmreg_access, .type = ARM_CP_NOP },
 912     /* Since we don't implement any events, writing to PMSELR is UNPREDICTABLE.
 913      * We choose to RAZ/WI.
 914      */
 915     { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5,
 916       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
 917       .accessfn = pmreg_access },
 918 #ifndef CONFIG_USER_ONLY
 919     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
 920       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_IO,
 921       .readfn = pmccntr_read, .writefn = pmccntr_write32,
 922       .accessfn = pmreg_access },
 923     { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
 924       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
 925       .access = PL0_RW, .accessfn = pmreg_access,
 926       .type = ARM_CP_IO,
 927       .readfn = pmccntr_read, .writefn = pmccntr_write, },
 928 #endif
 929     { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
 930       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
 931       .writefn = pmccfiltr_write,
 932       .access = PL0_RW, .accessfn = pmreg_access,
 933       .type = ARM_CP_IO,
 934       .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
 935       .resetvalue = 0, },
 936     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
 937       .access = PL0_RW,
 938       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmxevtyper),
 939       .accessfn = pmreg_access, .writefn = pmxevtyper_write,
 940       .raw_writefn = raw_write },
 941     /* Unimplemented, RAZ/WI. */
 942     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
 943       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
 944       .accessfn = pmreg_access },
 945     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
 946       .access = PL0_R | PL1_RW,
 947       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
 948       .resetvalue = 0,
 949       .writefn = pmuserenr_write, .raw_writefn = raw_write },
 950     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
 951       .access = PL1_RW,
 952       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
 953       .resetvalue = 0,
 954       .writefn = pmintenset_write, .raw_writefn = raw_write },
 955     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
 956       .access = PL1_RW, .type = ARM_CP_ALIAS,
 957       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
 958       .resetvalue = 0, .writefn = pmintenclr_write, },
 959     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
 960       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
 961       .access = PL1_RW, .writefn = vbar_write,
 962       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
 963                              offsetof(CPUARMState, cp15.vbar_ns) },
 964       .resetvalue = 0 },
 965     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
 966       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
 967       .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_RAW },
 968     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
 969       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
 970       .access = PL1_RW, .writefn = csselr_write, .resetvalue = 0,
 971       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
 972                              offsetof(CPUARMState, cp15.csselr_ns) } },
 973     /* Auxiliary ID register: this actually has an IMPDEF value but for now
 974      * just RAZ for all cores:
 975      */
 976     { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
 977       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
 978       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
 979     /* Auxiliary fault status registers: these also are IMPDEF, and we
 980      * choose to RAZ/WI for all cores.
 981      */
 982     { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
 983       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
 984       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
 985     { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
 986       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
 987       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
 988     /* MAIR can just read-as-written because we don't implement caches
 989      * and so don't need to care about memory attributes.
 990      */
 991     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
 992       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
 993       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
 994       .resetvalue = 0 },
 995     /* For non-long-descriptor page tables these are PRRR and NMRR;
 996      * regardless they still act as reads-as-written for QEMU.
 997      * The override is necessary because of the overly-broad TLB_LOCKDOWN
 998      * definition.
 999      */
1000      /* MAIR0/1 are defined separately from their 64-bit counterpart which
1001       * allows them to assign the correct fieldoffset based on the endianness
1002       * handled in the field definitions.
1003       */
1004     { .name = "MAIR0", .state = ARM_CP_STATE_AA32, .type = ARM_CP_OVERRIDE,
1005       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0, .access = PL1_RW,
1006       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
1007                              offsetof(CPUARMState, cp15.mair0_ns) },
1008       .resetfn = arm_cp_reset_ignore },
1009     { .name = "MAIR1", .state = ARM_CP_STATE_AA32, .type = ARM_CP_OVERRIDE,
1010       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1, .access = PL1_RW,
1011       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
1012                              offsetof(CPUARMState, cp15.mair1_ns) },
1013       .resetfn = arm_cp_reset_ignore },
1014     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
1015       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
1016       .type = ARM_CP_NO_RAW, .access = PL1_R, .readfn = isr_read },
1017     /* 32 bit ITLB invalidates */
1018     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
1019       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1020     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
1021       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1022     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
1023       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1024     /* 32 bit DTLB invalidates */
1025     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
1026       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1027     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
1028       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1029     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
1030       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1031     /* 32 bit TLB invalidates */
1032     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
1033       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1034     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
1035       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1036     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
1037       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1038     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
1039       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
1040     REGINFO_SENTINEL
1041 };
1042
1043 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
1044     /* 32 bit TLB invalidates, Inner Shareable */
1045     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
1046       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_is_write },
1047     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
1048       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
1049     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
1050       .type = ARM_CP_NO_RAW, .access = PL1_W,
1051       .writefn = tlbiasid_is_write },
1052     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
1053       .type = ARM_CP_NO_RAW, .access = PL1_W,
1054       .writefn = tlbimvaa_is_write },
1055     REGINFO_SENTINEL
1056 };
1057
1058 static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1059                         uint64_t value)
1060 {
1061     value &= 1;
1062     env->teecr = value;
1063 }
1064
1065 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri)
1066 {
1067     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
1068         return CP_ACCESS_TRAP;
1069     }
1070     return CP_ACCESS_OK;
1071 }
1072
1073 static const ARMCPRegInfo t2ee_cp_reginfo[] = {
1074     { .name = "TEECR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 6, .opc2 = 0,
1075       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, teecr),
1076       .resetvalue = 0,
1077       .writefn = teecr_write },
1078     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
1079       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
1080       .accessfn = teehbr_access, .resetvalue = 0 },
1081     REGINFO_SENTINEL
1082 };
1083
1084 static const ARMCPRegInfo v6k_cp_reginfo[] = {
1085     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
1086       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
1087       .access = PL0_RW,
1088       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
1089     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
1090       .access = PL0_RW,
1091       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
1092                              offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
1093       .resetfn = arm_cp_reset_ignore },
1094     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
1095       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
1096       .access = PL0_R|PL1_W,
1097       .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
1098       .resetvalue = 0},
1099     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
1100       .access = PL0_R|PL1_W,
1101       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
1102                              offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
1103       .resetfn = arm_cp_reset_ignore },
1104     { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
1105       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
1106       .access = PL1_RW,
1107       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
1108     { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
1109       .access = PL1_RW,
1110       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
1111                              offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
1112       .resetvalue = 0 },
1113     REGINFO_SENTINEL
1114 };
1115
1116 #ifndef CONFIG_USER_ONLY
1117
1118 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
1119 {
1120     /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero */
1121     if (arm_current_el(env) == 0 && !extract32(env->cp15.c14_cntkctl, 0, 2)) {
1122         return CP_ACCESS_TRAP;
1123     }
1124     return CP_ACCESS_OK;
1125 }
1126
1127 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx)
1128 {
1129     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
1130     if (arm_current_el(env) == 0 &&
1131         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
1132         return CP_ACCESS_TRAP;
1133     }
1134     return CP_ACCESS_OK;
1135 }
1136
1137 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx)
1138 {
1139     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
1140      * EL0[PV]TEN is zero.
1141      */
1142     if (arm_current_el(env) == 0 &&
1143         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
1144         return CP_ACCESS_TRAP;
1145     }
1146     return CP_ACCESS_OK;
1147 }
1148
1149 static CPAccessResult gt_pct_access(CPUARMState *env,
1150                                          const ARMCPRegInfo *ri)
1151 {
1152     return gt_counter_access(env, GTIMER_PHYS);
1153 }
1154
1155 static CPAccessResult gt_vct_access(CPUARMState *env,
1156                                          const ARMCPRegInfo *ri)
1157 {
1158     return gt_counter_access(env, GTIMER_VIRT);
1159 }
1160
1161 static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
1162 {
1163     return gt_timer_access(env, GTIMER_PHYS);
1164 }
1165
1166 static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
1167 {
1168     return gt_timer_access(env, GTIMER_VIRT);
1169 }
1170
1171 static uint64_t gt_get_countervalue(CPUARMState *env)
1172 {
1173     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / GTIMER_SCALE;
1174 }
1175
1176 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
1177 {
1178     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
1179
1180     if (gt->ctl & 1) {
1181         /* Timer enabled: calculate and set current ISTATUS, irq, and
1182          * reset timer to when ISTATUS next has to change
1183          */
1184         uint64_t count = gt_get_countervalue(&cpu->env);
1185         /* Note that this must be unsigned 64 bit arithmetic: */
1186         int istatus = count >= gt->cval;
1187         uint64_t nexttick;
1188
1189         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
1190         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1191                      (istatus && !(gt->ctl & 2)));
1192         if (istatus) {
1193             /* Next transition is when count rolls back over to zero */
1194             nexttick = UINT64_MAX;
1195         } else {
1196             /* Next transition is when we hit cval */
1197             nexttick = gt->cval;
1198         }
1199         /* Note that the desired next expiry time might be beyond the
1200          * signed-64-bit range of a QEMUTimer -- in this case we just
1201          * set the timer for as far in the future as possible. When the
1202          * timer expires we will reset the timer for any remaining period.
1203          */
1204         if (nexttick > INT64_MAX / GTIMER_SCALE) {
1205             nexttick = INT64_MAX / GTIMER_SCALE;
1206         }
1207         timer_mod(cpu->gt_timer[timeridx], nexttick);
1208     } else {
1209         /* Timer disabled: ISTATUS and timer output always clear */
1210         gt->ctl &= ~4;
1211         qemu_set_irq(cpu->gt_timer_outputs[timeridx], 0);
1212         timer_del(cpu->gt_timer[timeridx]);
1213     }
1214 }
1215
1216 static void gt_cnt_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1217 {
1218     ARMCPU *cpu = arm_env_get_cpu(env);
1219     int timeridx = ri->opc1 & 1;
1220
1221     timer_del(cpu->gt_timer[timeridx]);
1222 }
1223
1224 static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
1225 {
1226     return gt_get_countervalue(env);
1227 }
1228
1229 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1230                           uint64_t value)
1231 {
1232     int timeridx = ri->opc1 & 1;
1233
1234     env->cp15.c14_timer[timeridx].cval = value;
1235     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1236 }
1237
1238 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1239 {
1240     int timeridx = ri->crm & 1;
1241
1242     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
1243                       gt_get_countervalue(env));
1244 }
1245
1246 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1247                           uint64_t value)
1248 {
1249     int timeridx = ri->crm & 1;
1250
1251     env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) +
1252         + sextract64(value, 0, 32);
1253     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1254 }
1255
1256 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1257                          uint64_t value)
1258 {
1259     ARMCPU *cpu = arm_env_get_cpu(env);
1260     int timeridx = ri->crm & 1;
1261     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
1262
1263     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
1264     if ((oldval ^ value) & 1) {
1265         /* Enable toggled */
1266         gt_recalc_timer(cpu, timeridx);
1267     } else if ((oldval ^ value) & 2) {
1268         /* IMASK toggled: don't need to recalculate,
1269          * just set the interrupt line based on ISTATUS
1270          */
1271         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1272                      (oldval & 4) && !(value & 2));
1273     }
1274 }
1275
1276 void arm_gt_ptimer_cb(void *opaque)
1277 {
1278     ARMCPU *cpu = opaque;
1279
1280     gt_recalc_timer(cpu, GTIMER_PHYS);
1281 }
1282
1283 void arm_gt_vtimer_cb(void *opaque)
1284 {
1285     ARMCPU *cpu = opaque;
1286
1287     gt_recalc_timer(cpu, GTIMER_VIRT);
1288 }
1289
1290 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1291     /* Note that CNTFRQ is purely reads-as-written for the benefit
1292      * of software; writing it doesn't actually change the timer frequency.
1293      * Our reset value matches the fixed frequency we implement the timer at.
1294      */
1295     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
1296       .type = ARM_CP_ALIAS,
1297       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1298       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
1299       .resetfn = arm_cp_reset_ignore,
1300     },
1301     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
1302       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
1303       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1304       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
1305       .resetvalue = (1000 * 1000 * 1000) / GTIMER_SCALE,
1306     },
1307     /* overall control: mostly access permissions */
1308     { .name = "CNTKCTL", .state = ARM_CP_STATE_BOTH,
1309       .opc0 = 3, .opc1 = 0, .crn = 14, .crm = 1, .opc2 = 0,
1310       .access = PL1_RW,
1311       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntkctl),
1312       .resetvalue = 0,
1313     },
1314     /* per-timer control */
1315     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
1316       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1317       .accessfn = gt_ptimer_access,
1318       .fieldoffset = offsetoflow32(CPUARMState,
1319                                    cp15.c14_timer[GTIMER_PHYS].ctl),
1320       .resetfn = arm_cp_reset_ignore,
1321       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1322     },
1323     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
1324       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
1325       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1326       .accessfn = gt_ptimer_access,
1327       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
1328       .resetvalue = 0,
1329       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1330     },
1331     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
1332       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1333       .accessfn = gt_vtimer_access,
1334       .fieldoffset = offsetoflow32(CPUARMState,
1335                                    cp15.c14_timer[GTIMER_VIRT].ctl),
1336       .resetfn = arm_cp_reset_ignore,
1337       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1338     },
1339     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
1340       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
1341       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1342       .accessfn = gt_vtimer_access,
1343       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
1344       .resetvalue = 0,
1345       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1346     },
1347     /* TimerValue views: a 32 bit downcounting view of the underlying state */
1348     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
1349       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1350       .accessfn = gt_ptimer_access,
1351       .readfn = gt_tval_read, .writefn = gt_tval_write,
1352     },
1353     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1354       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
1355       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1356       .readfn = gt_tval_read, .writefn = gt_tval_write,
1357     },
1358     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
1359       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1360       .accessfn = gt_vtimer_access,
1361       .readfn = gt_tval_read, .writefn = gt_tval_write,
1362     },
1363     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1364       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
1365       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1366       .readfn = gt_tval_read, .writefn = gt_tval_write,
1367     },
1368     /* The counter itself */
1369     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
1370       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1371       .accessfn = gt_pct_access,
1372       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1373     },
1374     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
1375       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
1376       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1377       .accessfn = gt_pct_access,
1378       .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
1379     },
1380     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
1381       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1382       .accessfn = gt_vct_access,
1383       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1384     },
1385     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
1386       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
1387       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1388       .accessfn = gt_vct_access,
1389       .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
1390     },
1391     /* Comparison value, indicating when the timer goes off */
1392     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
1393       .access = PL1_RW | PL0_R,
1394       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1395       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1396       .accessfn = gt_ptimer_access, .resetfn = arm_cp_reset_ignore,
1397       .writefn = gt_cval_write, .raw_writefn = raw_write,
1398     },
1399     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1400       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
1401       .access = PL1_RW | PL0_R,
1402       .type = ARM_CP_IO,
1403       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1404       .resetvalue = 0, .accessfn = gt_vtimer_access,
1405       .writefn = gt_cval_write, .raw_writefn = raw_write,
1406     },
1407     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
1408       .access = PL1_RW | PL0_R,
1409       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1410       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1411       .accessfn = gt_vtimer_access, .resetfn = arm_cp_reset_ignore,
1412       .writefn = gt_cval_write, .raw_writefn = raw_write,
1413     },
1414     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1415       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
1416       .access = PL1_RW | PL0_R,
1417       .type = ARM_CP_IO,
1418       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1419       .resetvalue = 0, .accessfn = gt_vtimer_access,
1420       .writefn = gt_cval_write, .raw_writefn = raw_write,
1421     },
1422     REGINFO_SENTINEL
1423 };
1424
1425 #else
1426 /* In user-mode none of the generic timer registers are accessible,
1427  * and their implementation depends on QEMU_CLOCK_VIRTUAL and qdev gpio outputs,
1428  * so instead just don't register any of them.
1429  */
1430 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1431     REGINFO_SENTINEL
1432 };
1433
1434 #endif
1435
1436 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1437 {
1438     if (arm_feature(env, ARM_FEATURE_LPAE)) {
1439         raw_write(env, ri, value);
1440     } else if (arm_feature(env, ARM_FEATURE_V7)) {
1441         raw_write(env, ri, value & 0xfffff6ff);
1442     } else {
1443         raw_write(env, ri, value & 0xfffff1ff);
1444     }
1445 }
1446
1447 #ifndef CONFIG_USER_ONLY
1448 /* get_phys_addr() isn't present for user-mode-only targets */
1449
1450 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri)
1451 {
1452     if (ri->opc2 & 4) {
1453         /* Other states are only available with TrustZone; in
1454          * a non-TZ implementation these registers don't exist
1455          * at all, which is an Uncategorized trap. This underdecoding
1456          * is safe because the reginfo is NO_RAW.
1457          */
1458         return CP_ACCESS_TRAP_UNCATEGORIZED;
1459     }
1460     return CP_ACCESS_OK;
1461 }
1462
1463 static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
1464                              int access_type, ARMMMUIdx mmu_idx)
1465 {
1466     hwaddr phys_addr;
1467     target_ulong page_size;
1468     int prot;
1469     int ret;
1470     uint64_t par64;
1471     MemTxAttrs attrs = {};
1472
1473     ret = get_phys_addr(env, value, access_type, mmu_idx,
1474                         &phys_addr, &attrs, &prot, &page_size);
1475     if (extended_addresses_enabled(env)) {
1476         /* ret is a DFSR/IFSR value for the long descriptor
1477          * translation table format, but with WnR always clear.
1478          * Convert it to a 64-bit PAR.
1479          */
1480         par64 = (1 << 11); /* LPAE bit always set */
1481         if (ret == 0) {
1482             par64 |= phys_addr & ~0xfffULL;
1483             if (!attrs.secure) {
1484                 par64 |= (1 << 9); /* NS */
1485             }
1486             /* We don't set the ATTR or SH fields in the PAR. */
1487         } else {
1488             par64 |= 1; /* F */
1489             par64 |= (ret & 0x3f) << 1; /* FS */
1490             /* Note that S2WLK and FSTAGE are always zero, because we don't
1491              * implement virtualization and therefore there can't be a stage 2
1492              * fault.
1493              */
1494         }
1495     } else {
1496         /* ret is a DFSR/IFSR value for the short descriptor
1497          * translation table format (with WnR always clear).
1498          * Convert it to a 32-bit PAR.
1499          */
1500         if (ret == 0) {
1501             /* We do not set any attribute bits in the PAR */
1502             if (page_size == (1 << 24)
1503                 && arm_feature(env, ARM_FEATURE_V7)) {
1504                 par64 = (phys_addr & 0xff000000) | (1 << 1);
1505             } else {
1506                 par64 = phys_addr & 0xfffff000;
1507             }
1508             if (!attrs.secure) {
1509                 par64 |= (1 << 9); /* NS */
1510             }
1511         } else {
1512             par64 = ((ret & (1 << 10)) >> 5) | ((ret & (1 << 12)) >> 6) |
1513                     ((ret & 0xf) << 1) | 1;
1514         }
1515     }
1516     return par64;
1517 }
1518
1519 static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1520 {
1521     int access_type = ri->opc2 & 1;
1522     uint64_t par64;
1523     ARMMMUIdx mmu_idx;
1524     int el = arm_current_el(env);
1525     bool secure = arm_is_secure_below_el3(env);
1526
1527     switch (ri->opc2 & 6) {
1528     case 0:
1529         /* stage 1 current state PL1: ATS1CPR, ATS1CPW */
1530         switch (el) {
1531         case 3:
1532             mmu_idx = ARMMMUIdx_S1E3;
1533             break;
1534         case 2:
1535             mmu_idx = ARMMMUIdx_S1NSE1;
1536             break;
1537         case 1:
1538             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
1539             break;
1540         default:
1541             g_assert_not_reached();
1542         }
1543         break;
1544     case 2:
1545         /* stage 1 current state PL0: ATS1CUR, ATS1CUW */
1546         switch (el) {
1547         case 3:
1548             mmu_idx = ARMMMUIdx_S1SE0;
1549             break;
1550         case 2:
1551             mmu_idx = ARMMMUIdx_S1NSE0;
1552             break;
1553         case 1:
1554             mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
1555             break;
1556         default:
1557             g_assert_not_reached();
1558         }
1559         break;
1560     case 4:
1561         /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */
1562         mmu_idx = ARMMMUIdx_S12NSE1;
1563         break;
1564     case 6:
1565         /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */
1566         mmu_idx = ARMMMUIdx_S12NSE0;
1567         break;
1568     default:
1569         g_assert_not_reached();
1570     }
1571
1572     par64 = do_ats_write(env, value, access_type, mmu_idx);
1573
1574     A32_BANKED_CURRENT_REG_SET(env, par, par64);
1575 }
1576
1577 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
1578                         uint64_t value)
1579 {
1580     int access_type = ri->opc2 & 1;
1581     ARMMMUIdx mmu_idx;
1582     int secure = arm_is_secure_below_el3(env);
1583
1584     switch (ri->opc2 & 6) {
1585     case 0:
1586         switch (ri->opc1) {
1587         case 0: /* AT S1E1R, AT S1E1W */
1588             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
1589             break;
1590         case 4: /* AT S1E2R, AT S1E2W */
1591             mmu_idx = ARMMMUIdx_S1E2;
1592             break;
1593         case 6: /* AT S1E3R, AT S1E3W */
1594             mmu_idx = ARMMMUIdx_S1E3;
1595             break;
1596         default:
1597             g_assert_not_reached();
1598         }
1599         break;
1600     case 2: /* AT S1E0R, AT S1E0W */
1601         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
1602         break;
1603     case 4: /* AT S12E1R, AT S12E1W */
1604         mmu_idx = ARMMMUIdx_S12NSE1;
1605         break;
1606     case 6: /* AT S12E0R, AT S12E0W */
1607         mmu_idx = ARMMMUIdx_S12NSE0;
1608         break;
1609     default:
1610         g_assert_not_reached();
1611     }
1612
1613     env->cp15.par_el[1] = do_ats_write(env, value, access_type, mmu_idx);
1614 }
1615 #endif
1616
1617 static const ARMCPRegInfo vapa_cp_reginfo[] = {
1618     { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
1619       .access = PL1_RW, .resetvalue = 0,
1620       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
1621                              offsetoflow32(CPUARMState, cp15.par_ns) },
1622       .writefn = par_write },
1623 #ifndef CONFIG_USER_ONLY
1624     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
1625       .access = PL1_W, .accessfn = ats_access,
1626       .writefn = ats_write, .type = ARM_CP_NO_RAW },
1627 #endif
1628     REGINFO_SENTINEL
1629 };
1630
1631 /* Return basic MPU access permission bits.  */
1632 static uint32_t simple_mpu_ap_bits(uint32_t val)
1633 {
1634     uint32_t ret;
1635     uint32_t mask;
1636     int i;
1637     ret = 0;
1638     mask = 3;
1639     for (i = 0; i < 16; i += 2) {
1640         ret |= (val >> i) & mask;
1641         mask <<= 2;
1642     }
1643     return ret;
1644 }
1645
1646 /* Pad basic MPU access permission bits to extended format.  */
1647 static uint32_t extended_mpu_ap_bits(uint32_t val)
1648 {
1649     uint32_t ret;
1650     uint32_t mask;
1651     int i;
1652     ret = 0;
1653     mask = 3;
1654     for (i = 0; i < 16; i += 2) {
1655         ret |= (val & mask) << i;
1656         mask <<= 2;
1657     }
1658     return ret;
1659 }
1660
1661 static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
1662                                  uint64_t value)
1663 {
1664     env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
1665 }
1666
1667 static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
1668 {
1669     return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
1670 }
1671
1672 static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
1673                                  uint64_t value)
1674 {
1675     env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
1676 }
1677
1678 static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
1679 {
1680     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
1681 }
1682
1683 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
1684     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
1685       .access = PL1_RW, .type = ARM_CP_ALIAS,
1686       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
1687       .resetvalue = 0,
1688       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
1689     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
1690       .access = PL1_RW, .type = ARM_CP_ALIAS,
1691       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
1692       .resetvalue = 0,
1693       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
1694     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
1695       .access = PL1_RW,
1696       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
1697       .resetvalue = 0, },
1698     { .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
1699       .access = PL1_RW,
1700       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
1701       .resetvalue = 0, },
1702     { .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
1703       .access = PL1_RW,
1704       .fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
1705     { .name = "ICACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
1706       .access = PL1_RW,
1707       .fieldoffset = offsetof(CPUARMState, cp15.c2_insn), .resetvalue = 0, },
1708     /* Protection region base and size registers */
1709     { .name = "946_PRBS0", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0,
1710       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1711       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[0]) },
1712     { .name = "946_PRBS1", .cp = 15, .crn = 6, .crm = 1, .opc1 = 0,
1713       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1714       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[1]) },
1715     { .name = "946_PRBS2", .cp = 15, .crn = 6, .crm = 2, .opc1 = 0,
1716       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1717       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[2]) },
1718     { .name = "946_PRBS3", .cp = 15, .crn = 6, .crm = 3, .opc1 = 0,
1719       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1720       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[3]) },
1721     { .name = "946_PRBS4", .cp = 15, .crn = 6, .crm = 4, .opc1 = 0,
1722       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1723       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[4]) },
1724     { .name = "946_PRBS5", .cp = 15, .crn = 6, .crm = 5, .opc1 = 0,
1725       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1726       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[5]) },
1727     { .name = "946_PRBS6", .cp = 15, .crn = 6, .crm = 6, .opc1 = 0,
1728       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1729       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[6]) },
1730     { .name = "946_PRBS7", .cp = 15, .crn = 6, .crm = 7, .opc1 = 0,
1731       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1732       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[7]) },
1733     REGINFO_SENTINEL
1734 };
1735
1736 static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
1737                                  uint64_t value)
1738 {
1739     TCR *tcr = raw_ptr(env, ri);
1740     int maskshift = extract32(value, 0, 3);
1741
1742     if (!arm_feature(env, ARM_FEATURE_V8)) {
1743         if (arm_feature(env, ARM_FEATURE_LPAE) && (value & TTBCR_EAE)) {
1744             /* Pre ARMv8 bits [21:19], [15:14] and [6:3] are UNK/SBZP when
1745              * using Long-desciptor translation table format */
1746             value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
1747         } else if (arm_feature(env, ARM_FEATURE_EL3)) {
1748             /* In an implementation that includes the Security Extensions
1749              * TTBCR has additional fields PD0 [4] and PD1 [5] for
1750              * Short-descriptor translation table format.
1751              */
1752             value &= TTBCR_PD1 | TTBCR_PD0 | TTBCR_N;
1753         } else {
1754             value &= TTBCR_N;
1755         }
1756     }
1757
1758     /* Update the masks corresponding to the the TCR bank being written
1759      * Note that we always calculate mask and base_mask, but
1760      * they are only used for short-descriptor tables (ie if EAE is 0);
1761      * for long-descriptor tables the TCR fields are used differently
1762      * and the mask and base_mask values are meaningless.
1763      */
1764     tcr->raw_tcr = value;
1765     tcr->mask = ~(((uint32_t)0xffffffffu) >> maskshift);
1766     tcr->base_mask = ~((uint32_t)0x3fffu >> maskshift);
1767 }
1768
1769 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1770                              uint64_t value)
1771 {
1772     ARMCPU *cpu = arm_env_get_cpu(env);
1773
1774     if (arm_feature(env, ARM_FEATURE_LPAE)) {
1775         /* With LPAE the TTBCR could result in a change of ASID
1776          * via the TTBCR.A1 bit, so do a TLB flush.
1777          */
1778         tlb_flush(CPU(cpu), 1);
1779     }
1780     vmsa_ttbcr_raw_write(env, ri, value);
1781 }
1782
1783 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1784 {
1785     TCR *tcr = raw_ptr(env, ri);
1786
1787     /* Reset both the TCR as well as the masks corresponding to the bank of
1788      * the TCR being reset.
1789      */
1790     tcr->raw_tcr = 0;
1791     tcr->mask = 0;
1792     tcr->base_mask = 0xffffc000u;
1793 }
1794
1795 static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri,
1796                                uint64_t value)
1797 {
1798     ARMCPU *cpu = arm_env_get_cpu(env);
1799     TCR *tcr = raw_ptr(env, ri);
1800
1801     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
1802     tlb_flush(CPU(cpu), 1);
1803     tcr->raw_tcr = value;
1804 }
1805
1806 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1807                             uint64_t value)
1808 {
1809     /* 64 bit accesses to the TTBRs can change the ASID and so we
1810      * must flush the TLB.
1811      */
1812     if (cpreg_field_is_64bit(ri)) {
1813         ARMCPU *cpu = arm_env_get_cpu(env);
1814
1815         tlb_flush(CPU(cpu), 1);
1816     }
1817     raw_write(env, ri, value);
1818 }
1819
1820 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
1821     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
1822       .access = PL1_RW, .type = ARM_CP_ALIAS,
1823       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
1824                              offsetoflow32(CPUARMState, cp15.dfsr_ns) },
1825       .resetfn = arm_cp_reset_ignore, },
1826     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
1827       .access = PL1_RW, .resetvalue = 0,
1828       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
1829                              offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
1830     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
1831       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
1832       .access = PL1_RW,
1833       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
1834     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
1835       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
1836       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
1837       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
1838                              offsetof(CPUARMState, cp15.ttbr0_ns) } },
1839     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
1840       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
1841       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
1842       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
1843                              offsetof(CPUARMState, cp15.ttbr1_ns) } },
1844     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
1845       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
1846       .access = PL1_RW, .writefn = vmsa_tcr_el1_write,
1847       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
1848       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
1849     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
1850       .access = PL1_RW, .type = ARM_CP_ALIAS, .writefn = vmsa_ttbcr_write,
1851       .resetfn = arm_cp_reset_ignore, .raw_writefn = vmsa_ttbcr_raw_write,
1852       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
1853                              offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
1854     { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
1855       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
1856       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
1857       .resetvalue = 0, },
1858     { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
1859       .access = PL1_RW, .resetvalue = 0,
1860       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
1861                              offsetof(CPUARMState, cp15.dfar_ns) } },
1862     REGINFO_SENTINEL
1863 };
1864
1865 static void omap_ticonfig_write(CPUARMState *env, const ARMCPRegInfo *ri,
1866                                 uint64_t value)
1867 {
1868     env->cp15.c15_ticonfig = value & 0xe7;
1869     /* The OS_TYPE bit in this register changes the reported CPUID! */
1870     env->cp15.c0_cpuid = (value & (1 << 5)) ?
1871         ARM_CPUID_TI915T : ARM_CPUID_TI925T;
1872 }
1873
1874 static void omap_threadid_write(CPUARMState *env, const ARMCPRegInfo *ri,
1875                                 uint64_t value)
1876 {
1877     env->cp15.c15_threadid = value & 0xffff;
1878 }
1879
1880 static void omap_wfi_write(CPUARMState *env, const ARMCPRegInfo *ri,
1881                            uint64_t value)
1882 {
1883     /* Wait-for-interrupt (deprecated) */
1884     cpu_interrupt(CPU(arm_env_get_cpu(env)), CPU_INTERRUPT_HALT);
1885 }
1886
1887 static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
1888                                   uint64_t value)
1889 {
1890     /* On OMAP there are registers indicating the max/min index of dcache lines
1891      * containing a dirty line; cache flush operations have to reset these.
1892      */
1893     env->cp15.c15_i_max = 0x000;
1894     env->cp15.c15_i_min = 0xff0;
1895 }
1896
1897 static const ARMCPRegInfo omap_cp_reginfo[] = {
1898     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
1899       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
1900       .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
1901       .resetvalue = 0, },
1902     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
1903       .access = PL1_RW, .type = ARM_CP_NOP },
1904     { .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
1905       .access = PL1_RW,
1906       .fieldoffset = offsetof(CPUARMState, cp15.c15_ticonfig), .resetvalue = 0,
1907       .writefn = omap_ticonfig_write },
1908     { .name = "IMAX", .cp = 15, .crn = 15, .crm = 2, .opc1 = 0, .opc2 = 0,
1909       .access = PL1_RW,
1910       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_max), .resetvalue = 0, },
1911     { .name = "IMIN", .cp = 15, .crn = 15, .crm = 3, .opc1 = 0, .opc2 = 0,
1912       .access = PL1_RW, .resetvalue = 0xff0,
1913       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_min) },
1914     { .name = "THREADID", .cp = 15, .crn = 15, .crm = 4, .opc1 = 0, .opc2 = 0,
1915       .access = PL1_RW,
1916       .fieldoffset = offsetof(CPUARMState, cp15.c15_threadid), .resetvalue = 0,
1917       .writefn = omap_threadid_write },
1918     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
1919       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
1920       .type = ARM_CP_NO_RAW,
1921       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
1922     /* TODO: Peripheral port remap register:
1923      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
1924      * base address at $rn & ~0xfff and map size of 0x200 << ($rn & 0xfff),
1925      * when MMU is off.
1926      */
1927     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
1928       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
1929       .type = ARM_CP_OVERRIDE | ARM_CP_NO_RAW,
1930       .writefn = omap_cachemaint_write },
1931     { .name = "C9", .cp = 15, .crn = 9,
1932       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
1933       .type = ARM_CP_CONST | ARM_CP_OVERRIDE, .resetvalue = 0 },
1934     REGINFO_SENTINEL
1935 };
1936
1937 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
1938                               uint64_t value)
1939 {
1940     env->cp15.c15_cpar = value & 0x3fff;
1941 }
1942
1943 static const ARMCPRegInfo xscale_cp_reginfo[] = {
1944     { .name = "XSCALE_CPAR",
1945       .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
1946       .fieldoffset = offsetof(CPUARMState, cp15.c15_cpar), .resetvalue = 0,
1947       .writefn = xscale_cpar_write, },
1948     { .name = "XSCALE_AUXCR",
1949       .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW,
1950       .fieldoffset = offsetof(CPUARMState, cp15.c1_xscaleauxcr),
1951       .resetvalue = 0, },
1952     /* XScale specific cache-lockdown: since we have no cache we NOP these
1953      * and hope the guest does not really rely on cache behaviour.
1954      */
1955     { .name = "XSCALE_LOCK_ICACHE_LINE",
1956       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
1957       .access = PL1_W, .type = ARM_CP_NOP },
1958     { .name = "XSCALE_UNLOCK_ICACHE",
1959       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
1960       .access = PL1_W, .type = ARM_CP_NOP },
1961     { .name = "XSCALE_DCACHE_LOCK",
1962       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 0,
1963       .access = PL1_RW, .type = ARM_CP_NOP },
1964     { .name = "XSCALE_UNLOCK_DCACHE",
1965       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 1,
1966       .access = PL1_W, .type = ARM_CP_NOP },
1967     REGINFO_SENTINEL
1968 };
1969
1970 static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
1971     /* RAZ/WI the whole crn=15 space, when we don't have a more specific
1972      * implementation of this implementation-defined space.
1973      * Ideally this should eventually disappear in favour of actually
1974      * implementing the correct behaviour for all cores.
1975      */
1976     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
1977       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
1978       .access = PL1_RW,
1979       .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
1980       .resetvalue = 0 },
1981     REGINFO_SENTINEL
1982 };
1983
1984 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
1985     /* Cache status: RAZ because we have no cache so it's always clean */
1986     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
1987       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
1988       .resetvalue = 0 },
1989     REGINFO_SENTINEL
1990 };
1991
1992 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
1993     /* We never have a a block transfer operation in progress */
1994     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
1995       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
1996       .resetvalue = 0 },
1997     /* The cache ops themselves: these all NOP for QEMU */
1998     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
1999       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2000     { .name = "IDCR", .cp = 15, .crm = 6, .opc1 = 0,
2001       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2002     { .name = "CDCR", .cp = 15, .crm = 12, .opc1 = 0,
2003       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2004     { .name = "PIR", .cp = 15, .crm = 12, .opc1 = 1,
2005       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2006     { .name = "PDR", .cp = 15, .crm = 12, .opc1 = 2,
2007       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2008     { .name = "CIDCR", .cp = 15, .crm = 14, .opc1 = 0,
2009       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2010     REGINFO_SENTINEL
2011 };
2012
2013 static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
2014     /* The cache test-and-clean instructions always return (1 << 30)
2015      * to indicate that there are no dirty cache lines.
2016      */
2017     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
2018       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2019       .resetvalue = (1 << 30) },
2020     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
2021       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2022       .resetvalue = (1 << 30) },
2023     REGINFO_SENTINEL
2024 };
2025
2026 static const ARMCPRegInfo strongarm_cp_reginfo[] = {
2027     /* Ignore ReadBuffer accesses */
2028     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
2029       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
2030       .access = PL1_RW, .resetvalue = 0,
2031       .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_RAW },
2032     REGINFO_SENTINEL
2033 };
2034
2035 static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2036 {
2037     CPUState *cs = CPU(arm_env_get_cpu(env));
2038     uint32_t mpidr = cs->cpu_index;
2039     /* We don't support setting cluster ID ([8..11]) (known as Aff1
2040      * in later ARM ARM versions), or any of the higher affinity level fields,
2041      * so these bits always RAZ.
2042      */
2043     if (arm_feature(env, ARM_FEATURE_V7MP)) {
2044         mpidr |= (1U << 31);
2045         /* Cores which are uniprocessor (non-coherent)
2046          * but still implement the MP extensions set
2047          * bit 30. (For instance, A9UP.) However we do
2048          * not currently model any of those cores.
2049          */
2050     }
2051     return mpidr;
2052 }
2053
2054 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
2055     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
2056       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
2057       .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_RAW },
2058     REGINFO_SENTINEL
2059 };
2060
2061 static const ARMCPRegInfo lpae_cp_reginfo[] = {
2062     /* NOP AMAIR0/1: the override is because these clash with the rather
2063      * broadly specified TLB_LOCKDOWN entry in the generic cp_reginfo.
2064      */
2065     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
2066       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
2067       .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
2068       .resetvalue = 0 },
2069     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
2070     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
2071       .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
2072       .resetvalue = 0 },
2073     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
2074       .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
2075       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
2076                              offsetof(CPUARMState, cp15.par_ns)} },
2077     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
2078       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2079       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
2080                              offsetof(CPUARMState, cp15.ttbr0_ns) },
2081       .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
2082     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
2083       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2084       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
2085                              offsetof(CPUARMState, cp15.ttbr1_ns) },
2086       .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
2087     REGINFO_SENTINEL
2088 };
2089
2090 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2091 {
2092     return vfp_get_fpcr(env);
2093 }
2094
2095 static void aa64_fpcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2096                             uint64_t value)
2097 {
2098     vfp_set_fpcr(env, value);
2099 }
2100
2101 static uint64_t aa64_fpsr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2102 {
2103     return vfp_get_fpsr(env);
2104 }
2105
2106 static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2107                             uint64_t value)
2108 {
2109     vfp_set_fpsr(env, value);
2110 }
2111
2112 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri)
2113 {
2114     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
2115         return CP_ACCESS_TRAP;
2116     }
2117     return CP_ACCESS_OK;
2118 }
2119
2120 static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
2121                             uint64_t value)
2122 {
2123     env->daif = value & PSTATE_DAIF;
2124 }
2125
2126 static CPAccessResult aa64_cacheop_access(CPUARMState *env,
2127                                           const ARMCPRegInfo *ri)
2128 {
2129     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
2130      * SCTLR_EL1.UCI is set.
2131      */
2132     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) {
2133         return CP_ACCESS_TRAP;
2134     }
2135     return CP_ACCESS_OK;
2136 }
2137
2138 /* See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
2139  * Page D4-1736 (DDI0487A.b)
2140  */
2141
2142 static void tlbi_aa64_va_write(CPUARMState *env, const ARMCPRegInfo *ri,
2143                                uint64_t value)
2144 {
2145     /* Invalidate by VA (AArch64 version) */
2146     ARMCPU *cpu = arm_env_get_cpu(env);
2147     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2148
2149     tlb_flush_page(CPU(cpu), pageaddr);
2150 }
2151
2152 static void tlbi_aa64_vaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
2153                                 uint64_t value)
2154 {
2155     /* Invalidate by VA, all ASIDs (AArch64 version) */
2156     ARMCPU *cpu = arm_env_get_cpu(env);
2157     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2158
2159     tlb_flush_page(CPU(cpu), pageaddr);
2160 }
2161
2162 static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
2163                                  uint64_t value)
2164 {
2165     /* Invalidate by ASID (AArch64 version) */
2166     ARMCPU *cpu = arm_env_get_cpu(env);
2167     int asid = extract64(value, 48, 16);
2168     tlb_flush(CPU(cpu), asid == 0);
2169 }
2170
2171 static void tlbi_aa64_va_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2172                                   uint64_t value)
2173 {
2174     CPUState *other_cs;
2175     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2176
2177     CPU_FOREACH(other_cs) {
2178         tlb_flush_page(other_cs, pageaddr);
2179     }
2180 }
2181
2182 static void tlbi_aa64_vaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2183                                   uint64_t value)
2184 {
2185     CPUState *other_cs;
2186     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2187
2188     CPU_FOREACH(other_cs) {
2189         tlb_flush_page(other_cs, pageaddr);
2190     }
2191 }
2192
2193 static void tlbi_aa64_asid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2194                                   uint64_t value)
2195 {
2196     CPUState *other_cs;
2197     int asid = extract64(value, 48, 16);
2198
2199     CPU_FOREACH(other_cs) {
2200         tlb_flush(other_cs, asid == 0);
2201     }
2202 }
2203
2204 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri)
2205 {
2206     /* We don't implement EL2, so the only control on DC ZVA is the
2207      * bit in the SCTLR which can prohibit access for EL0.
2208      */
2209     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
2210         return CP_ACCESS_TRAP;
2211     }
2212     return CP_ACCESS_OK;
2213 }
2214
2215 static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
2216 {
2217     ARMCPU *cpu = arm_env_get_cpu(env);
2218     int dzp_bit = 1 << 4;
2219
2220     /* DZP indicates whether DC ZVA access is allowed */
2221     if (aa64_zva_access(env, NULL) == CP_ACCESS_OK) {
2222         dzp_bit = 0;
2223     }
2224     return cpu->dcz_blocksize | dzp_bit;
2225 }
2226
2227 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
2228 {
2229     if (!(env->pstate & PSTATE_SP)) {
2230         /* Access to SP_EL0 is undefined if it's being used as
2231          * the stack pointer.
2232          */
2233         return CP_ACCESS_TRAP_UNCATEGORIZED;
2234     }
2235     return CP_ACCESS_OK;
2236 }
2237
2238 static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
2239 {
2240     return env->pstate & PSTATE_SP;
2241 }
2242
2243 static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
2244 {
2245     update_spsel(env, val);
2246 }
2247
2248 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2249                         uint64_t value)
2250 {
2251     ARMCPU *cpu = arm_env_get_cpu(env);
2252
2253     if (raw_read(env, ri) == value) {
2254         /* Skip the TLB flush if nothing actually changed; Linux likes
2255          * to do a lot of pointless SCTLR writes.
2256          */
2257         return;
2258     }
2259
2260     raw_write(env, ri, value);
2261     /* ??? Lots of these bits are not implemented.  */
2262     /* This may enable/disable the MMU, so do a TLB flush.  */
2263     tlb_flush(CPU(cpu), 1);
2264 }
2265
2266 static const ARMCPRegInfo v8_cp_reginfo[] = {
2267     /* Minimal set of EL0-visible registers. This will need to be expanded
2268      * significantly for system emulation of AArch64 CPUs.
2269      */
2270     { .name = "NZCV", .state = ARM_CP_STATE_AA64,
2271       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
2272       .access = PL0_RW, .type = ARM_CP_NZCV },
2273     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
2274       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
2275       .type = ARM_CP_NO_RAW,
2276       .access = PL0_RW, .accessfn = aa64_daif_access,
2277       .fieldoffset = offsetof(CPUARMState, daif),
2278       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
2279     { .name = "FPCR", .state = ARM_CP_STATE_AA64,
2280       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
2281       .access = PL0_RW, .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
2282     { .name = "FPSR", .state = ARM_CP_STATE_AA64,
2283       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
2284       .access = PL0_RW, .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
2285     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
2286       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
2287       .access = PL0_R, .type = ARM_CP_NO_RAW,
2288       .readfn = aa64_dczid_read },
2289     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
2290       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
2291       .access = PL0_W, .type = ARM_CP_DC_ZVA,
2292 #ifndef CONFIG_USER_ONLY
2293       /* Avoid overhead of an access check that always passes in user-mode */
2294       .accessfn = aa64_zva_access,
2295 #endif
2296     },
2297     { .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
2298       .opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
2299       .access = PL1_R, .type = ARM_CP_CURRENTEL },
2300     /* Cache ops: all NOPs since we don't emulate caches */
2301     { .name = "IC_IALLUIS", .state = ARM_CP_STATE_AA64,
2302       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
2303       .access = PL1_W, .type = ARM_CP_NOP },
2304     { .name = "IC_IALLU", .state = ARM_CP_STATE_AA64,
2305       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
2306       .access = PL1_W, .type = ARM_CP_NOP },
2307     { .name = "IC_IVAU", .state = ARM_CP_STATE_AA64,
2308       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 5, .opc2 = 1,
2309       .access = PL0_W, .type = ARM_CP_NOP,
2310       .accessfn = aa64_cacheop_access },
2311     { .name = "DC_IVAC", .state = ARM_CP_STATE_AA64,
2312       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
2313       .access = PL1_W, .type = ARM_CP_NOP },
2314     { .name = "DC_ISW", .state = ARM_CP_STATE_AA64,
2315       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
2316       .access = PL1_W, .type = ARM_CP_NOP },
2317     { .name = "DC_CVAC", .state = ARM_CP_STATE_AA64,
2318       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 1,
2319       .access = PL0_W, .type = ARM_CP_NOP,
2320       .accessfn = aa64_cacheop_access },
2321     { .name = "DC_CSW", .state = ARM_CP_STATE_AA64,
2322       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
2323       .access = PL1_W, .type = ARM_CP_NOP },
2324     { .name = "DC_CVAU", .state = ARM_CP_STATE_AA64,
2325       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 11, .opc2 = 1,
2326       .access = PL0_W, .type = ARM_CP_NOP,
2327       .accessfn = aa64_cacheop_access },
2328     { .name = "DC_CIVAC", .state = ARM_CP_STATE_AA64,
2329       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 1,
2330       .access = PL0_W, .type = ARM_CP_NOP,
2331       .accessfn = aa64_cacheop_access },
2332     { .name = "DC_CISW", .state = ARM_CP_STATE_AA64,
2333       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
2334       .access = PL1_W, .type = ARM_CP_NOP },
2335     /* TLBI operations */
2336     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
2337       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
2338       .access = PL1_W, .type = ARM_CP_NO_RAW,
2339       .writefn = tlbiall_is_write },
2340     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
2341       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
2342       .access = PL1_W, .type = ARM_CP_NO_RAW,
2343       .writefn = tlbi_aa64_va_is_write },
2344     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
2345       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
2346       .access = PL1_W, .type = ARM_CP_NO_RAW,
2347       .writefn = tlbi_aa64_asid_is_write },
2348     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
2349       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
2350       .access = PL1_W, .type = ARM_CP_NO_RAW,
2351       .writefn = tlbi_aa64_vaa_is_write },
2352     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
2353       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
2354       .access = PL1_W, .type = ARM_CP_NO_RAW,
2355       .writefn = tlbi_aa64_va_is_write },
2356     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
2357       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
2358       .access = PL1_W, .type = ARM_CP_NO_RAW,
2359       .writefn = tlbi_aa64_vaa_is_write },
2360     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
2361       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
2362       .access = PL1_W, .type = ARM_CP_NO_RAW,
2363       .writefn = tlbiall_write },
2364     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
2365       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
2366       .access = PL1_W, .type = ARM_CP_NO_RAW,
2367       .writefn = tlbi_aa64_va_write },
2368     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
2369       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
2370       .access = PL1_W, .type = ARM_CP_NO_RAW,
2371       .writefn = tlbi_aa64_asid_write },
2372     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
2373       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
2374       .access = PL1_W, .type = ARM_CP_NO_RAW,
2375       .writefn = tlbi_aa64_vaa_write },
2376     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
2377       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
2378       .access = PL1_W, .type = ARM_CP_NO_RAW,
2379       .writefn = tlbi_aa64_va_write },
2380     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
2381       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
2382       .access = PL1_W, .type = ARM_CP_NO_RAW,
2383       .writefn = tlbi_aa64_vaa_write },
2384 #ifndef CONFIG_USER_ONLY
2385     /* 64 bit address translation operations */
2386     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
2387       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
2388       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2389     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
2390       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
2391       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2392     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
2393       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
2394       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2395     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
2396       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
2397       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
2398 #endif
2399     /* TLB invalidate last level of translation table walk */
2400     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
2401       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
2402     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
2403       .type = ARM_CP_NO_RAW, .access = PL1_W,
2404       .writefn = tlbimvaa_is_write },
2405     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
2406       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
2407     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
2408       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
2409     /* 32 bit cache operations */
2410     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
2411       .type = ARM_CP_NOP, .access = PL1_W },
2412     { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
2413       .type = ARM_CP_NOP, .access = PL1_W },
2414     { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
2415       .type = ARM_CP_NOP, .access = PL1_W },
2416     { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
2417       .type = ARM_CP_NOP, .access = PL1_W },
2418     { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
2419       .type = ARM_CP_NOP, .access = PL1_W },
2420     { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
2421       .type = ARM_CP_NOP, .access = PL1_W },
2422     { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
2423       .type = ARM_CP_NOP, .access = PL1_W },
2424     { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
2425       .type = ARM_CP_NOP, .access = PL1_W },
2426     { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
2427       .type = ARM_CP_NOP, .access = PL1_W },
2428     { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
2429       .type = ARM_CP_NOP, .access = PL1_W },
2430     { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
2431       .type = ARM_CP_NOP, .access = PL1_W },
2432     { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
2433       .type = ARM_CP_NOP, .access = PL1_W },
2434     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
2435       .type = ARM_CP_NOP, .access = PL1_W },
2436     /* MMU Domain access control / MPU write buffer control */
2437     { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
2438       .access = PL1_RW, .resetvalue = 0,
2439       .writefn = dacr_write, .raw_writefn = raw_write,
2440       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
2441                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
2442     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
2443       .type = ARM_CP_ALIAS,
2444       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
2445       .access = PL1_RW,
2446       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
2447     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
2448       .type = ARM_CP_ALIAS,
2449       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
2450       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[1]) },
2451     /* We rely on the access checks not allowing the guest to write to the
2452      * state field when SPSel indicates that it's being used as the stack
2453      * pointer.
2454      */
2455     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
2456       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
2457       .access = PL1_RW, .accessfn = sp_el0_access,
2458       .type = ARM_CP_ALIAS,
2459       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
2460     { .name = "SP_EL1", .state = ARM_CP_STATE_AA64,
2461       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 1, .opc2 = 0,
2462       .access = PL2_RW, .type = ARM_CP_ALIAS,
2463       .fieldoffset = offsetof(CPUARMState, sp_el[1]) },
2464     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
2465       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
2466       .type = ARM_CP_NO_RAW,
2467       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
2468     REGINFO_SENTINEL
2469 };
2470
2471 /* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
2472 static const ARMCPRegInfo v8_el3_no_el2_cp_reginfo[] = {
2473     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
2474       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
2475       .access = PL2_RW,
2476       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
2477     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
2478       .type = ARM_CP_NO_RAW,
2479       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
2480       .access = PL2_RW,
2481       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
2482     REGINFO_SENTINEL
2483 };
2484
2485 static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
2486 {
2487     ARMCPU *cpu = arm_env_get_cpu(env);
2488     uint64_t valid_mask = HCR_MASK;
2489
2490     if (arm_feature(env, ARM_FEATURE_EL3)) {
2491         valid_mask &= ~HCR_HCD;
2492     } else {
2493         valid_mask &= ~HCR_TSC;
2494     }
2495
2496     /* Clear RES0 bits.  */
2497     value &= valid_mask;
2498
2499     /* These bits change the MMU setup:
2500      * HCR_VM enables stage 2 translation
2501      * HCR_PTW forbids certain page-table setups
2502      * HCR_DC Disables stage1 and enables stage2 translation
2503      */
2504     if ((raw_read(env, ri) ^ value) & (HCR_VM | HCR_PTW | HCR_DC)) {
2505         tlb_flush(CPU(cpu), 1);
2506     }
2507     raw_write(env, ri, value);
2508 }
2509
2510 static const ARMCPRegInfo v8_el2_cp_reginfo[] = {
2511     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
2512       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
2513       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
2514       .writefn = hcr_write },
2515     { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
2516       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
2517       .access = PL2_RW, .resetvalue = 0,
2518       .writefn = dacr_write, .raw_writefn = raw_write,
2519       .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
2520     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
2521       .type = ARM_CP_ALIAS,
2522       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
2523       .access = PL2_RW,
2524       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
2525     { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
2526       .type = ARM_CP_ALIAS,
2527       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
2528       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
2529     { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
2530       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
2531       .access = PL2_RW, .resetvalue = 0,
2532       .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
2533     { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
2534       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
2535       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
2536     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
2537       .type = ARM_CP_ALIAS,
2538       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
2539       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[6]) },
2540     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
2541       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
2542       .access = PL2_RW, .writefn = vbar_write,
2543       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
2544       .resetvalue = 0 },
2545     { .name = "SP_EL2", .state = ARM_CP_STATE_AA64,
2546       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 1, .opc2 = 0,
2547       .access = PL3_RW, .type = ARM_CP_ALIAS,
2548       .fieldoffset = offsetof(CPUARMState, sp_el[2]) },
2549     REGINFO_SENTINEL
2550 };
2551
2552 static const ARMCPRegInfo el3_cp_reginfo[] = {
2553     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
2554       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
2555       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
2556       .resetvalue = 0, .writefn = scr_write },
2557     { .name = "SCR",  .type = ARM_CP_ALIAS,
2558       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
2559       .access = PL3_RW, .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
2560       .resetfn = arm_cp_reset_ignore, .writefn = scr_write },
2561     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
2562       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
2563       .access = PL3_RW, .resetvalue = 0,
2564       .fieldoffset = offsetof(CPUARMState, cp15.sder) },
2565     { .name = "SDER",
2566       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
2567       .access = PL3_RW, .resetvalue = 0,
2568       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
2569       /* TODO: Implement NSACR trapping of secure EL1 accesses to EL3 */
2570     { .name = "NSACR", .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
2571       .access = PL3_W | PL1_R, .resetvalue = 0,
2572       .fieldoffset = offsetof(CPUARMState, cp15.nsacr) },
2573     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
2574       .access = PL3_RW, .writefn = vbar_write, .resetvalue = 0,
2575       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
2576     { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
2577       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
2578       .access = PL3_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
2579       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]) },
2580     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
2581       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
2582       .access = PL3_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
2583       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
2584     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
2585       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
2586       .access = PL3_RW, .writefn = vmsa_tcr_el1_write,
2587       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
2588       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
2589     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
2590       .type = ARM_CP_ALIAS,
2591       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
2592       .access = PL3_RW,
2593       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
2594     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
2595       .type = ARM_CP_ALIAS,
2596       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
2597       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
2598     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
2599       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
2600       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
2601     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
2602       .type = ARM_CP_ALIAS,
2603       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
2604       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[7]) },
2605     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
2606       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
2607       .access = PL3_RW, .writefn = vbar_write,
2608       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
2609       .resetvalue = 0 },
2610     REGINFO_SENTINEL
2611 };
2612
2613 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
2614 {
2615     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
2616      * but the AArch32 CTR has its own reginfo struct)
2617      */
2618     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
2619         return CP_ACCESS_TRAP;
2620     }
2621     return CP_ACCESS_OK;
2622 }
2623
2624 static const ARMCPRegInfo debug_cp_reginfo[] = {
2625     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
2626      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
2627      * unlike DBGDRAR it is never accessible from EL0.
2628      * DBGDSAR is deprecated and must RAZ from v8 anyway, so it has no AArch64
2629      * accessor.
2630      */
2631     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
2632       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2633     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
2634       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
2635       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2636     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
2637       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2638     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
2639     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
2640       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
2641       .access = PL1_RW,
2642       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
2643       .resetvalue = 0 },
2644     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
2645      * We don't implement the configurable EL0 access.
2646      */
2647     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
2648       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
2649       .type = ARM_CP_ALIAS,
2650       .access = PL1_R,
2651       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
2652       .resetfn = arm_cp_reset_ignore },
2653     /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
2654     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
2655       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
2656       .access = PL1_W, .type = ARM_CP_NOP },
2657     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
2658     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
2659       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
2660       .access = PL1_RW, .type = ARM_CP_NOP },
2661     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
2662      * implement vector catch debug events yet.
2663      */
2664     { .name = "DBGVCR",
2665       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
2666       .access = PL1_RW, .type = ARM_CP_NOP },
2667     REGINFO_SENTINEL
2668 };
2669
2670 static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
2671     /* 64 bit access versions of the (dummy) debug registers */
2672     { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
2673       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
2674     { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
2675       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
2676     REGINFO_SENTINEL
2677 };
2678
2679 void hw_watchpoint_update(ARMCPU *cpu, int n)
2680 {
2681     CPUARMState *env = &cpu->env;
2682     vaddr len = 0;
2683     vaddr wvr = env->cp15.dbgwvr[n];
2684     uint64_t wcr = env->cp15.dbgwcr[n];
2685     int mask;
2686     int flags = BP_CPU | BP_STOP_BEFORE_ACCESS;
2687
2688     if (env->cpu_watchpoint[n]) {
2689         cpu_watchpoint_remove_by_ref(CPU(cpu), env->cpu_watchpoint[n]);
2690         env->cpu_watchpoint[n] = NULL;
2691     }
2692
2693     if (!extract64(wcr, 0, 1)) {
2694         /* E bit clear : watchpoint disabled */
2695         return;
2696     }
2697
2698     switch (extract64(wcr, 3, 2)) {
2699     case 0:
2700         /* LSC 00 is reserved and must behave as if the wp is disabled */
2701         return;
2702     case 1:
2703         flags |= BP_MEM_READ;
2704         break;
2705     case 2:
2706         flags |= BP_MEM_WRITE;
2707         break;
2708     case 3:
2709         flags |= BP_MEM_ACCESS;
2710         break;
2711     }
2712
2713     /* Attempts to use both MASK and BAS fields simultaneously are
2714      * CONSTRAINED UNPREDICTABLE; we opt to ignore BAS in this case,
2715      * thus generating a watchpoint for every byte in the masked region.
2716      */
2717     mask = extract64(wcr, 24, 4);
2718     if (mask == 1 || mask == 2) {
2719         /* Reserved values of MASK; we must act as if the mask value was
2720          * some non-reserved value, or as if the watchpoint were disabled.
2721          * We choose the latter.
2722          */
2723         return;
2724     } else if (mask) {
2725         /* Watchpoint covers an aligned area up to 2GB in size */
2726         len = 1ULL << mask;
2727         /* If masked bits in WVR are not zero it's CONSTRAINED UNPREDICTABLE
2728          * whether the watchpoint fires when the unmasked bits match; we opt
2729          * to generate the exceptions.
2730          */
2731         wvr &= ~(len - 1);
2732     } else {
2733         /* Watchpoint covers bytes defined by the byte address select bits */
2734         int bas = extract64(wcr, 5, 8);
2735         int basstart;
2736
2737         if (bas == 0) {
2738             /* This must act as if the watchpoint is disabled */
2739             return;
2740         }
2741
2742         if (extract64(wvr, 2, 1)) {
2743             /* Deprecated case of an only 4-aligned address. BAS[7:4] are
2744              * ignored, and BAS[3:0] define which bytes to watch.
2745              */
2746             bas &= 0xf;
2747         }
2748         /* The BAS bits are supposed to be programmed to indicate a contiguous
2749          * range of bytes. Otherwise it is CONSTRAINED UNPREDICTABLE whether
2750          * we fire for each byte in the word/doubleword addressed by the WVR.
2751          * We choose to ignore any non-zero bits after the first range of 1s.
2752          */
2753         basstart = ctz32(bas);
2754         len = cto32(bas >> basstart);
2755         wvr += basstart;
2756     }
2757
2758     cpu_watchpoint_insert(CPU(cpu), wvr, len, flags,
2759                           &env->cpu_watchpoint[n]);
2760 }
2761
2762 void hw_watchpoint_update_all(ARMCPU *cpu)
2763 {
2764     int i;
2765     CPUARMState *env = &cpu->env;
2766
2767     /* Completely clear out existing QEMU watchpoints and our array, to
2768      * avoid possible stale entries following migration load.
2769      */
2770     cpu_watchpoint_remove_all(CPU(cpu), BP_CPU);
2771     memset(env->cpu_watchpoint, 0, sizeof(env->cpu_watchpoint));
2772
2773     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_watchpoint); i++) {
2774         hw_watchpoint_update(cpu, i);
2775     }
2776 }
2777
2778 static void dbgwvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2779                          uint64_t value)
2780 {
2781     ARMCPU *cpu = arm_env_get_cpu(env);
2782     int i = ri->crm;
2783
2784     /* Bits [63:49] are hardwired to the value of bit [48]; that is, the
2785      * register reads and behaves as if values written are sign extended.
2786      * Bits [1:0] are RES0.
2787      */
2788     value = sextract64(value, 0, 49) & ~3ULL;
2789
2790     raw_write(env, ri, value);
2791     hw_watchpoint_update(cpu, i);
2792 }
2793
2794 static void dbgwcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2795                          uint64_t value)
2796 {
2797     ARMCPU *cpu = arm_env_get_cpu(env);
2798     int i = ri->crm;
2799
2800     raw_write(env, ri, value);
2801     hw_watchpoint_update(cpu, i);
2802 }
2803
2804 void hw_breakpoint_update(ARMCPU *cpu, int n)
2805 {
2806     CPUARMState *env = &cpu->env;
2807     uint64_t bvr = env->cp15.dbgbvr[n];
2808     uint64_t bcr = env->cp15.dbgbcr[n];
2809     vaddr addr;
2810     int bt;
2811     int flags = BP_CPU;
2812
2813     if (env->cpu_breakpoint[n]) {
2814         cpu_breakpoint_remove_by_ref(CPU(cpu), env->cpu_breakpoint[n]);
2815         env->cpu_breakpoint[n] = NULL;
2816     }
2817
2818     if (!extract64(bcr, 0, 1)) {
2819         /* E bit clear : watchpoint disabled */
2820         return;
2821     }
2822
2823     bt = extract64(bcr, 20, 4);
2824
2825     switch (bt) {
2826     case 4: /* unlinked address mismatch (reserved if AArch64) */
2827     case 5: /* linked address mismatch (reserved if AArch64) */
2828         qemu_log_mask(LOG_UNIMP,
2829                       "arm: address mismatch breakpoint types not implemented");
2830         return;
2831     case 0: /* unlinked address match */
2832     case 1: /* linked address match */
2833     {
2834         /* Bits [63:49] are hardwired to the value of bit [48]; that is,
2835          * we behave as if the register was sign extended. Bits [1:0] are
2836          * RES0. The BAS field is used to allow setting breakpoints on 16
2837          * bit wide instructions; it is CONSTRAINED UNPREDICTABLE whether
2838          * a bp will fire if the addresses covered by the bp and the addresses
2839          * covered by the insn overlap but the insn doesn't start at the
2840          * start of the bp address range. We choose to require the insn and
2841          * the bp to have the same address. The constraints on writing to
2842          * BAS enforced in dbgbcr_write mean we have only four cases:
2843          *  0b0000  => no breakpoint
2844          *  0b0011  => breakpoint on addr
2845          *  0b1100  => breakpoint on addr + 2
2846          *  0b1111  => breakpoint on addr
2847          * See also figure D2-3 in the v8 ARM ARM (DDI0487A.c).
2848          */
2849         int bas = extract64(bcr, 5, 4);
2850         addr = sextract64(bvr, 0, 49) & ~3ULL;
2851         if (bas == 0) {
2852             return;
2853         }
2854         if (bas == 0xc) {
2855             addr += 2;
2856         }
2857         break;
2858     }
2859     case 2: /* unlinked context ID match */
2860     case 8: /* unlinked VMID match (reserved if no EL2) */
2861     case 10: /* unlinked context ID and VMID match (reserved if no EL2) */
2862         qemu_log_mask(LOG_UNIMP,
2863                       "arm: unlinked context breakpoint types not implemented");
2864         return;
2865     case 9: /* linked VMID match (reserved if no EL2) */
2866     case 11: /* linked context ID and VMID match (reserved if no EL2) */
2867     case 3: /* linked context ID match */
2868     default:
2869         /* We must generate no events for Linked context matches (unless
2870          * they are linked to by some other bp/wp, which is handled in
2871          * updates for the linking bp/wp). We choose to also generate no events
2872          * for reserved values.
2873          */
2874         return;
2875     }
2876
2877     cpu_breakpoint_insert(CPU(cpu), addr, flags, &env->cpu_breakpoint[n]);
2878 }
2879
2880 void hw_breakpoint_update_all(ARMCPU *cpu)
2881 {
2882     int i;
2883     CPUARMState *env = &cpu->env;
2884
2885     /* Completely clear out existing QEMU breakpoints and our array, to
2886      * avoid possible stale entries following migration load.
2887      */
2888     cpu_breakpoint_remove_all(CPU(cpu), BP_CPU);
2889     memset(env->cpu_breakpoint, 0, sizeof(env->cpu_breakpoint));
2890
2891     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_breakpoint); i++) {
2892         hw_breakpoint_update(cpu, i);
2893     }
2894 }
2895
2896 static void dbgbvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2897                          uint64_t value)
2898 {
2899     ARMCPU *cpu = arm_env_get_cpu(env);
2900     int i = ri->crm;
2901
2902     raw_write(env, ri, value);
2903     hw_breakpoint_update(cpu, i);
2904 }
2905
2906 static void dbgbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2907                          uint64_t value)
2908 {
2909     ARMCPU *cpu = arm_env_get_cpu(env);
2910     int i = ri->crm;
2911
2912     /* BAS[3] is a read-only copy of BAS[2], and BAS[1] a read-only
2913      * copy of BAS[0].
2914      */
2915     value = deposit64(value, 6, 1, extract64(value, 5, 1));
2916     value = deposit64(value, 8, 1, extract64(value, 7, 1));
2917
2918     raw_write(env, ri, value);
2919     hw_breakpoint_update(cpu, i);
2920 }
2921
2922 static void define_debug_regs(ARMCPU *cpu)
2923 {
2924     /* Define v7 and v8 architectural debug registers.
2925      * These are just dummy implementations for now.
2926      */
2927     int i;
2928     int wrps, brps, ctx_cmps;
2929     ARMCPRegInfo dbgdidr = {
2930         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
2931         .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
2932     };
2933
2934     /* Note that all these register fields hold "number of Xs minus 1". */
2935     brps = extract32(cpu->dbgdidr, 24, 4);
2936     wrps = extract32(cpu->dbgdidr, 28, 4);
2937     ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
2938
2939     assert(ctx_cmps <= brps);
2940
2941     /* The DBGDIDR and ID_AA64DFR0_EL1 define various properties
2942      * of the debug registers such as number of breakpoints;
2943      * check that if they both exist then they agree.
2944      */
2945     if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
2946         assert(extract32(cpu->id_aa64dfr0, 12, 4) == brps);
2947         assert(extract32(cpu->id_aa64dfr0, 20, 4) == wrps);
2948         assert(extract32(cpu->id_aa64dfr0, 28, 4) == ctx_cmps);
2949     }
2950
2951     define_one_arm_cp_reg(cpu, &dbgdidr);
2952     define_arm_cp_regs(cpu, debug_cp_reginfo);
2953
2954     if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
2955         define_arm_cp_regs(cpu, debug_lpae_cp_reginfo);
2956     }
2957
2958     for (i = 0; i < brps + 1; i++) {
2959         ARMCPRegInfo dbgregs[] = {
2960             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
2961               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
2962               .access = PL1_RW,
2963               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
2964               .writefn = dbgbvr_write, .raw_writefn = raw_write
2965             },
2966             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
2967               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
2968               .access = PL1_RW,
2969               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
2970               .writefn = dbgbcr_write, .raw_writefn = raw_write
2971             },
2972             REGINFO_SENTINEL
2973         };
2974         define_arm_cp_regs(cpu, dbgregs);
2975     }
2976
2977     for (i = 0; i < wrps + 1; i++) {
2978         ARMCPRegInfo dbgregs[] = {
2979             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
2980               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
2981               .access = PL1_RW,
2982               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
2983               .writefn = dbgwvr_write, .raw_writefn = raw_write
2984             },
2985             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
2986               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
2987               .access = PL1_RW,
2988               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
2989               .writefn = dbgwcr_write, .raw_writefn = raw_write
2990             },
2991             REGINFO_SENTINEL
2992         };
2993         define_arm_cp_regs(cpu, dbgregs);
2994     }
2995 }
2996
2997 void register_cp_regs_for_features(ARMCPU *cpu)
2998 {
2999     /* Register all the coprocessor registers based on feature bits */
3000     CPUARMState *env = &cpu->env;
3001     if (arm_feature(env, ARM_FEATURE_M)) {
3002         /* M profile has no coprocessor registers */
3003         return;
3004     }
3005
3006     define_arm_cp_regs(cpu, cp_reginfo);
3007     if (!arm_feature(env, ARM_FEATURE_V8)) {
3008         /* Must go early as it is full of wildcards that may be
3009          * overridden by later definitions.
3010          */
3011         define_arm_cp_regs(cpu, not_v8_cp_reginfo);
3012     }
3013
3014     if (arm_feature(env, ARM_FEATURE_V6)) {
3015         /* The ID registers all have impdef reset values */
3016         ARMCPRegInfo v6_idregs[] = {
3017             { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
3018               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
3019               .access = PL1_R, .type = ARM_CP_CONST,
3020               .resetvalue = cpu->id_pfr0 },
3021             { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
3022               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
3023               .access = PL1_R, .type = ARM_CP_CONST,
3024               .resetvalue = cpu->id_pfr1 },
3025             { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
3026               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
3027               .access = PL1_R, .type = ARM_CP_CONST,
3028               .resetvalue = cpu->id_dfr0 },
3029             { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
3030               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
3031               .access = PL1_R, .type = ARM_CP_CONST,
3032               .resetvalue = cpu->id_afr0 },
3033             { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
3034               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
3035               .access = PL1_R, .type = ARM_CP_CONST,
3036               .resetvalue = cpu->id_mmfr0 },
3037             { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
3038               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
3039               .access = PL1_R, .type = ARM_CP_CONST,
3040               .resetvalue = cpu->id_mmfr1 },
3041             { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
3042               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
3043               .access = PL1_R, .type = ARM_CP_CONST,
3044               .resetvalue = cpu->id_mmfr2 },
3045             { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
3046               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
3047               .access = PL1_R, .type = ARM_CP_CONST,
3048               .resetvalue = cpu->id_mmfr3 },
3049             { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
3050               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
3051               .access = PL1_R, .type = ARM_CP_CONST,
3052               .resetvalue = cpu->id_isar0 },
3053             { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
3054               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
3055               .access = PL1_R, .type = ARM_CP_CONST,
3056               .resetvalue = cpu->id_isar1 },
3057             { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
3058               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
3059               .access = PL1_R, .type = ARM_CP_CONST,
3060               .resetvalue = cpu->id_isar2 },
3061             { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
3062               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
3063               .access = PL1_R, .type = ARM_CP_CONST,
3064               .resetvalue = cpu->id_isar3 },
3065             { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
3066               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
3067               .access = PL1_R, .type = ARM_CP_CONST,
3068               .resetvalue = cpu->id_isar4 },
3069             { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
3070               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
3071               .access = PL1_R, .type = ARM_CP_CONST,
3072               .resetvalue = cpu->id_isar5 },
3073             /* 6..7 are as yet unallocated and must RAZ */
3074             { .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
3075               .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
3076               .resetvalue = 0 },
3077             { .name = "ID_ISAR7", .cp = 15, .crn = 0, .crm = 2,
3078               .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
3079               .resetvalue = 0 },
3080             REGINFO_SENTINEL
3081         };
3082         define_arm_cp_regs(cpu, v6_idregs);
3083         define_arm_cp_regs(cpu, v6_cp_reginfo);
3084     } else {
3085         define_arm_cp_regs(cpu, not_v6_cp_reginfo);
3086     }
3087     if (arm_feature(env, ARM_FEATURE_V6K)) {
3088         define_arm_cp_regs(cpu, v6k_cp_reginfo);
3089     }
3090     if (arm_feature(env, ARM_FEATURE_V7MP)) {
3091         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
3092     }
3093     if (arm_feature(env, ARM_FEATURE_V7)) {
3094         /* v7 performance monitor control register: same implementor
3095          * field as main ID register, and we implement only the cycle
3096          * count register.
3097          */
3098 #ifndef CONFIG_USER_ONLY
3099         ARMCPRegInfo pmcr = {
3100             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
3101             .access = PL0_RW,
3102             .type = ARM_CP_IO | ARM_CP_ALIAS,
3103             .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
3104             .accessfn = pmreg_access, .writefn = pmcr_write,
3105             .raw_writefn = raw_write,
3106         };
3107         ARMCPRegInfo pmcr64 = {
3108             .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
3109             .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
3110             .access = PL0_RW, .accessfn = pmreg_access,
3111             .type = ARM_CP_IO,
3112             .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
3113             .resetvalue = cpu->midr & 0xff000000,
3114             .writefn = pmcr_write, .raw_writefn = raw_write,
3115         };
3116         define_one_arm_cp_reg(cpu, &pmcr);
3117         define_one_arm_cp_reg(cpu, &pmcr64);
3118 #endif
3119         ARMCPRegInfo clidr = {
3120             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
3121             .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
3122             .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->clidr
3123         };
3124         define_one_arm_cp_reg(cpu, &clidr);
3125         define_arm_cp_regs(cpu, v7_cp_reginfo);
3126         define_debug_regs(cpu);
3127     } else {
3128         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
3129     }
3130     if (arm_feature(env, ARM_FEATURE_V8)) {
3131         /* AArch64 ID registers, which all have impdef reset values */
3132         ARMCPRegInfo v8_idregs[] = {
3133             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
3134               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
3135               .access = PL1_R, .type = ARM_CP_CONST,
3136               .resetvalue = cpu->id_aa64pfr0 },
3137             { .name = "ID_AA64PFR1_EL1", .state = ARM_CP_STATE_AA64,
3138               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
3139               .access = PL1_R, .type = ARM_CP_CONST,
3140               .resetvalue = cpu->id_aa64pfr1},
3141             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
3142               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
3143               .access = PL1_R, .type = ARM_CP_CONST,
3144               /* We mask out the PMUVer field, because we don't currently
3145                * implement the PMU. Not advertising it prevents the guest
3146                * from trying to use it and getting UNDEFs on registers we
3147                * don't implement.
3148                */
3149               .resetvalue = cpu->id_aa64dfr0 & ~0xf00 },
3150             { .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
3151               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
3152               .access = PL1_R, .type = ARM_CP_CONST,
3153               .resetvalue = cpu->id_aa64dfr1 },
3154             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
3155               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
3156               .access = PL1_R, .type = ARM_CP_CONST,
3157               .resetvalue = cpu->id_aa64afr0 },
3158             { .name = "ID_AA64AFR1_EL1", .state = ARM_CP_STATE_AA64,
3159               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
3160               .access = PL1_R, .type = ARM_CP_CONST,
3161               .resetvalue = cpu->id_aa64afr1 },
3162             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
3163               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
3164               .access = PL1_R, .type = ARM_CP_CONST,
3165               .resetvalue = cpu->id_aa64isar0 },
3166             { .name = "ID_AA64ISAR1_EL1", .state = ARM_CP_STATE_AA64,
3167               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
3168               .access = PL1_R, .type = ARM_CP_CONST,
3169               .resetvalue = cpu->id_aa64isar1 },
3170             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
3171               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
3172               .access = PL1_R, .type = ARM_CP_CONST,
3173               .resetvalue = cpu->id_aa64mmfr0 },
3174             { .name = "ID_AA64MMFR1_EL1", .state = ARM_CP_STATE_AA64,
3175               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
3176               .access = PL1_R, .type = ARM_CP_CONST,
3177               .resetvalue = cpu->id_aa64mmfr1 },
3178             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
3179               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
3180               .access = PL1_R, .type = ARM_CP_CONST,
3181               .resetvalue = cpu->mvfr0 },
3182             { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
3183               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
3184               .access = PL1_R, .type = ARM_CP_CONST,
3185               .resetvalue = cpu->mvfr1 },
3186             { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
3187               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
3188               .access = PL1_R, .type = ARM_CP_CONST,
3189               .resetvalue = cpu->mvfr2 },
3190             REGINFO_SENTINEL
3191         };
3192         /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
3193         if (!arm_feature(env, ARM_FEATURE_EL3) &&
3194             !arm_feature(env, ARM_FEATURE_EL2)) {
3195             ARMCPRegInfo rvbar = {
3196                 .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
3197                 .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
3198                 .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
3199             };
3200             define_one_arm_cp_reg(cpu, &rvbar);
3201         }
3202         define_arm_cp_regs(cpu, v8_idregs);
3203         define_arm_cp_regs(cpu, v8_cp_reginfo);
3204     }
3205     if (arm_feature(env, ARM_FEATURE_EL2)) {
3206         define_arm_cp_regs(cpu, v8_el2_cp_reginfo);
3207         /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
3208         if (!arm_feature(env, ARM_FEATURE_EL3)) {
3209             ARMCPRegInfo rvbar = {
3210                 .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
3211                 .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
3212                 .type = ARM_CP_CONST, .access = PL2_R, .resetvalue = cpu->rvbar
3213             };
3214             define_one_arm_cp_reg(cpu, &rvbar);
3215         }
3216     } else {
3217         /* If EL2 is missing but higher ELs are enabled, we need to
3218          * register the no_el2 reginfos.
3219          */
3220         if (arm_feature(env, ARM_FEATURE_EL3)) {
3221             define_arm_cp_regs(cpu, v8_el3_no_el2_cp_reginfo);
3222         }
3223     }
3224     if (arm_feature(env, ARM_FEATURE_EL3)) {
3225         define_arm_cp_regs(cpu, el3_cp_reginfo);
3226         ARMCPRegInfo rvbar = {
3227             .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
3228             .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
3229             .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar
3230         };
3231         define_one_arm_cp_reg(cpu, &rvbar);
3232     }
3233     if (arm_feature(env, ARM_FEATURE_MPU)) {
3234         /* These are the MPU registers prior to PMSAv6. Any new
3235          * PMSA core later than the ARM946 will require that we
3236          * implement the PMSAv6 or PMSAv7 registers, which are
3237          * completely different.
3238          */
3239         assert(!arm_feature(env, ARM_FEATURE_V6));
3240         define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
3241     } else {
3242         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
3243     }
3244     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
3245         define_arm_cp_regs(cpu, t2ee_cp_reginfo);
3246     }
3247     if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
3248         define_arm_cp_regs(cpu, generic_timer_cp_reginfo);
3249     }
3250     if (arm_feature(env, ARM_FEATURE_VAPA)) {
3251         define_arm_cp_regs(cpu, vapa_cp_reginfo);
3252     }
3253     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
3254         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
3255     }
3256     if (arm_feature(env, ARM_FEATURE_CACHE_DIRTY_REG)) {
3257         define_arm_cp_regs(cpu, cache_dirty_status_cp_reginfo);
3258     }
3259     if (arm_feature(env, ARM_FEATURE_CACHE_BLOCK_OPS)) {
3260         define_arm_cp_regs(cpu, cache_block_ops_cp_reginfo);
3261     }
3262     if (arm_feature(env, ARM_FEATURE_OMAPCP)) {
3263         define_arm_cp_regs(cpu, omap_cp_reginfo);
3264     }
3265     if (arm_feature(env, ARM_FEATURE_STRONGARM)) {
3266         define_arm_cp_regs(cpu, strongarm_cp_reginfo);
3267     }
3268     if (arm_feature(env, ARM_FEATURE_XSCALE)) {
3269         define_arm_cp_regs(cpu, xscale_cp_reginfo);
3270     }
3271     if (arm_feature(env, ARM_FEATURE_DUMMY_C15_REGS)) {
3272         define_arm_cp_regs(cpu, dummy_c15_cp_reginfo);
3273     }
3274     if (arm_feature(env, ARM_FEATURE_LPAE)) {
3275         define_arm_cp_regs(cpu, lpae_cp_reginfo);
3276     }
3277     /* Slightly awkwardly, the OMAP and StrongARM cores need all of
3278      * cp15 crn=0 to be writes-ignored, whereas for other cores they should
3279      * be read-only (ie write causes UNDEF exception).
3280      */
3281     {
3282         ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
3283             /* Pre-v8 MIDR space.
3284              * Note that the MIDR isn't a simple constant register because
3285              * of the TI925 behaviour where writes to another register can
3286              * cause the MIDR value to change.
3287              *
3288              * Unimplemented registers in the c15 0 0 0 space default to
3289              * MIDR. Define MIDR first as this entire space, then CTR, TCMTR
3290              * and friends override accordingly.
3291              */
3292             { .name = "MIDR",
3293               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
3294               .access = PL1_R, .resetvalue = cpu->midr,
3295               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
3296               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
3297               .type = ARM_CP_OVERRIDE },
3298             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
3299             { .name = "DUMMY",
3300               .cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
3301               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3302             { .name = "DUMMY",
3303               .cp = 15, .crn = 0, .crm = 4, .opc1 = 0, .opc2 = CP_ANY,
3304               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3305             { .name = "DUMMY",
3306               .cp = 15, .crn = 0, .crm = 5, .opc1 = 0, .opc2 = CP_ANY,
3307               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3308             { .name = "DUMMY",
3309               .cp = 15, .crn = 0, .crm = 6, .opc1 = 0, .opc2 = CP_ANY,
3310               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3311             { .name = "DUMMY",
3312               .cp = 15, .crn = 0, .crm = 7, .opc1 = 0, .opc2 = CP_ANY,
3313               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3314             REGINFO_SENTINEL
3315         };
3316         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
3317             /* v8 MIDR -- the wildcard isn't necessary, and nor is the
3318              * variable-MIDR TI925 behaviour. Instead we have a single
3319              * (strictly speaking IMPDEF) alias of the MIDR, REVIDR.
3320              */
3321             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
3322               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
3323               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
3324             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
3325               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
3326               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
3327             REGINFO_SENTINEL
3328         };
3329         ARMCPRegInfo id_cp_reginfo[] = {
3330             /* These are common to v8 and pre-v8 */
3331             { .name = "CTR",
3332               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
3333               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
3334             { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
3335               .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
3336               .access = PL0_R, .accessfn = ctr_el0_access,
3337               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
3338             /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
3339             { .name = "TCMTR",
3340               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
3341               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3342             { .name = "TLBTR",
3343               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
3344               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3345             REGINFO_SENTINEL
3346         };
3347         ARMCPRegInfo crn0_wi_reginfo = {
3348             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
3349             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
3350             .type = ARM_CP_NOP | ARM_CP_OVERRIDE
3351         };
3352         if (arm_feature(env, ARM_FEATURE_OMAPCP) ||
3353             arm_feature(env, ARM_FEATURE_STRONGARM)) {
3354             ARMCPRegInfo *r;
3355             /* Register the blanket "writes ignored" value first to cover the
3356              * whole space. Then update the specific ID registers to allow write
3357              * access, so that they ignore writes rather than causing them to
3358              * UNDEF.
3359              */
3360             define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
3361             for (r = id_pre_v8_midr_cp_reginfo;
3362                  r->type != ARM_CP_SENTINEL; r++) {
3363                 r->access = PL1_RW;
3364             }
3365             for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
3366                 r->access = PL1_RW;
3367             }
3368         }
3369         if (arm_feature(env, ARM_FEATURE_V8)) {
3370             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
3371         } else {
3372             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
3373         }
3374         define_arm_cp_regs(cpu, id_cp_reginfo);
3375     }
3376
3377     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
3378         define_arm_cp_regs(cpu, mpidr_cp_reginfo);
3379     }
3380
3381     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
3382         ARMCPRegInfo auxcr = {
3383             .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
3384             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
3385             .access = PL1_RW, .type = ARM_CP_CONST,
3386             .resetvalue = cpu->reset_auxcr
3387         };
3388         define_one_arm_cp_reg(cpu, &auxcr);
3389     }
3390
3391     if (arm_feature(env, ARM_FEATURE_CBAR)) {
3392         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
3393             /* 32 bit view is [31:18] 0...0 [43:32]. */
3394             uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
3395                 | extract64(cpu->reset_cbar, 32, 12);
3396             ARMCPRegInfo cbar_reginfo[] = {
3397                 { .name = "CBAR",
3398                   .type = ARM_CP_CONST,
3399                   .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
3400                   .access = PL1_R, .resetvalue = cpu->reset_cbar },
3401                 { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
3402                   .type = ARM_CP_CONST,
3403                   .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
3404                   .access = PL1_R, .resetvalue = cbar32 },
3405                 REGINFO_SENTINEL
3406             };
3407             /* We don't implement a r/w 64 bit CBAR currently */
3408             assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
3409             define_arm_cp_regs(cpu, cbar_reginfo);
3410         } else {
3411             ARMCPRegInfo cbar = {
3412                 .name = "CBAR",
3413                 .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
3414                 .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
3415                 .fieldoffset = offsetof(CPUARMState,
3416                                         cp15.c15_config_base_address)
3417             };
3418             if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
3419                 cbar.access = PL1_R;
3420                 cbar.fieldoffset = 0;
3421                 cbar.type = ARM_CP_CONST;
3422             }
3423             define_one_arm_cp_reg(cpu, &cbar);
3424         }
3425     }
3426
3427     /* Generic registers whose values depend on the implementation */
3428     {
3429         ARMCPRegInfo sctlr = {
3430             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
3431             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
3432             .access = PL1_RW,
3433             .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
3434                                    offsetof(CPUARMState, cp15.sctlr_ns) },
3435             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
3436             .raw_writefn = raw_write,
3437         };
3438         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
3439             /* Normally we would always end the TB on an SCTLR write, but Linux
3440              * arch/arm/mach-pxa/sleep.S expects two instructions following
3441              * an MMU enable to execute from cache.  Imitate this behaviour.
3442              */
3443             sctlr.type |= ARM_CP_SUPPRESS_TB_END;
3444         }
3445         define_one_arm_cp_reg(cpu, &sctlr);
3446     }
3447 }
3448
3449 ARMCPU *cpu_arm_init(const char *cpu_model)
3450 {
3451     return ARM_CPU(cpu_generic_init(TYPE_ARM_CPU, cpu_model));
3452 }
3453
3454 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
3455 {
3456     CPUState *cs = CPU(cpu);
3457     CPUARMState *env = &cpu->env;
3458
3459     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
3460         gdb_register_coprocessor(cs, aarch64_fpu_gdb_get_reg,
3461                                  aarch64_fpu_gdb_set_reg,
3462                                  34, "aarch64-fpu.xml", 0);
3463     } else if (arm_feature(env, ARM_FEATURE_NEON)) {
3464         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3465                                  51, "arm-neon.xml", 0);
3466     } else if (arm_feature(env, ARM_FEATURE_VFP3)) {
3467         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3468                                  35, "arm-vfp3.xml", 0);
3469     } else if (arm_feature(env, ARM_FEATURE_VFP)) {
3470         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3471                                  19, "arm-vfp.xml", 0);
3472     }
3473 }
3474
3475 /* Sort alphabetically by type name, except for "any". */
3476 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
3477 {
3478     ObjectClass *class_a = (ObjectClass *)a;
3479     ObjectClass *class_b = (ObjectClass *)b;
3480     const char *name_a, *name_b;
3481
3482     name_a = object_class_get_name(class_a);
3483     name_b = object_class_get_name(class_b);
3484     if (strcmp(name_a, "any-" TYPE_ARM_CPU) == 0) {
3485         return 1;
3486     } else if (strcmp(name_b, "any-" TYPE_ARM_CPU) == 0) {
3487         return -1;
3488     } else {
3489         return strcmp(name_a, name_b);
3490     }
3491 }
3492
3493 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
3494 {
3495     ObjectClass *oc = data;
3496     CPUListState *s = user_data;
3497     const char *typename;
3498     char *name;
3499
3500     typename = object_class_get_name(oc);
3501     name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_ARM_CPU));
3502     (*s->cpu_fprintf)(s->file, "  %s\n",
3503                       name);
3504     g_free(name);
3505 }
3506
3507 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
3508 {
3509     CPUListState s = {
3510         .file = f,
3511         .cpu_fprintf = cpu_fprintf,
3512     };
3513     GSList *list;
3514
3515     list = object_class_get_list(TYPE_ARM_CPU, false);
3516     list = g_slist_sort(list, arm_cpu_list_compare);
3517     (*cpu_fprintf)(f, "Available CPUs:\n");
3518     g_slist_foreach(list, arm_cpu_list_entry, &s);
3519     g_slist_free(list);
3520 #ifdef CONFIG_KVM
3521     /* The 'host' CPU type is dynamically registered only if KVM is
3522      * enabled, so we have to special-case it here:
3523      */
3524     (*cpu_fprintf)(f, "  host (only available in KVM mode)\n");
3525 #endif
3526 }
3527
3528 static void arm_cpu_add_definition(gpointer data, gpointer user_data)
3529 {
3530     ObjectClass *oc = data;
3531     CpuDefinitionInfoList **cpu_list = user_data;
3532     CpuDefinitionInfoList *entry;
3533     CpuDefinitionInfo *info;
3534     const char *typename;
3535
3536     typename = object_class_get_name(oc);
3537     info = g_malloc0(sizeof(*info));
3538     info->name = g_strndup(typename,
3539                            strlen(typename) - strlen("-" TYPE_ARM_CPU));
3540
3541     entry = g_malloc0(sizeof(*entry));
3542     entry->value = info;
3543     entry->next = *cpu_list;
3544     *cpu_list = entry;
3545 }
3546
3547 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
3548 {
3549     CpuDefinitionInfoList *cpu_list = NULL;
3550     GSList *list;
3551
3552     list = object_class_get_list(TYPE_ARM_CPU, false);
3553     g_slist_foreach(list, arm_cpu_add_definition, &cpu_list);
3554     g_slist_free(list);
3555
3556     return cpu_list;
3557 }
3558
3559 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
3560                                    void *opaque, int state, int secstate,
3561                                    int crm, int opc1, int opc2)
3562 {
3563     /* Private utility function for define_one_arm_cp_reg_with_opaque():
3564      * add a single reginfo struct to the hash table.
3565      */
3566     uint32_t *key = g_new(uint32_t, 1);
3567     ARMCPRegInfo *r2 = g_memdup(r, sizeof(ARMCPRegInfo));
3568     int is64 = (r->type & ARM_CP_64BIT) ? 1 : 0;
3569     int ns = (secstate & ARM_CP_SECSTATE_NS) ? 1 : 0;
3570
3571     /* Reset the secure state to the specific incoming state.  This is
3572      * necessary as the register may have been defined with both states.
3573      */
3574     r2->secure = secstate;
3575
3576     if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
3577         /* Register is banked (using both entries in array).
3578          * Overwriting fieldoffset as the array is only used to define
3579          * banked registers but later only fieldoffset is used.
3580          */
3581         r2->fieldoffset = r->bank_fieldoffsets[ns];
3582     }
3583
3584     if (state == ARM_CP_STATE_AA32) {
3585         if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
3586             /* If the register is banked then we don't need to migrate or
3587              * reset the 32-bit instance in certain cases:
3588              *
3589              * 1) If the register has both 32-bit and 64-bit instances then we
3590              *    can count on the 64-bit instance taking care of the
3591              *    non-secure bank.
3592              * 2) If ARMv8 is enabled then we can count on a 64-bit version
3593              *    taking care of the secure bank.  This requires that separate
3594              *    32 and 64-bit definitions are provided.
3595              */
3596             if ((r->state == ARM_CP_STATE_BOTH && ns) ||
3597                 (arm_feature(&cpu->env, ARM_FEATURE_V8) && !ns)) {
3598                 r2->type |= ARM_CP_ALIAS;
3599                 r2->resetfn = arm_cp_reset_ignore;
3600             }
3601         } else if ((secstate != r->secure) && !ns) {
3602             /* The register is not banked so we only want to allow migration of
3603              * the non-secure instance.
3604              */
3605             r2->type |= ARM_CP_ALIAS;
3606             r2->resetfn = arm_cp_reset_ignore;
3607         }
3608
3609         if (r->state == ARM_CP_STATE_BOTH) {
3610             /* We assume it is a cp15 register if the .cp field is left unset.
3611              */
3612             if (r2->cp == 0) {
3613                 r2->cp = 15;
3614             }
3615
3616 #ifdef HOST_WORDS_BIGENDIAN
3617             if (r2->fieldoffset) {
3618                 r2->fieldoffset += sizeof(uint32_t);
3619             }
3620 #endif
3621         }
3622     }
3623     if (state == ARM_CP_STATE_AA64) {
3624         /* To allow abbreviation of ARMCPRegInfo
3625          * definitions, we treat cp == 0 as equivalent to
3626          * the value for "standard guest-visible sysreg".
3627          * STATE_BOTH definitions are also always "standard
3628          * sysreg" in their AArch64 view (the .cp value may
3629          * be non-zero for the benefit of the AArch32 view).
3630          */
3631         if (r->cp == 0 || r->state == ARM_CP_STATE_BOTH) {
3632             r2->cp = CP_REG_ARM64_SYSREG_CP;
3633         }
3634         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
3635                                   r2->opc0, opc1, opc2);
3636     } else {
3637         *key = ENCODE_CP_REG(r2->cp, is64, ns, r2->crn, crm, opc1, opc2);
3638     }
3639     if (opaque) {
3640         r2->opaque = opaque;
3641     }
3642     /* reginfo passed to helpers is correct for the actual access,
3643      * and is never ARM_CP_STATE_BOTH:
3644      */
3645     r2->state = state;
3646     /* Make sure reginfo passed to helpers for wildcarded regs
3647      * has the correct crm/opc1/opc2 for this reg, not CP_ANY:
3648      */
3649     r2->crm = crm;
3650     r2->opc1 = opc1;
3651     r2->opc2 = opc2;
3652     /* By convention, for wildcarded registers only the first
3653      * entry is used for migration; the others are marked as
3654      * ALIAS so we don't try to transfer the register
3655      * multiple times. Special registers (ie NOP/WFI) are
3656      * never migratable and not even raw-accessible.
3657      */
3658     if ((r->type & ARM_CP_SPECIAL)) {
3659         r2->type |= ARM_CP_NO_RAW;
3660     }
3661     if (((r->crm == CP_ANY) && crm != 0) ||
3662         ((r->opc1 == CP_ANY) && opc1 != 0) ||
3663         ((r->opc2 == CP_ANY) && opc2 != 0)) {
3664         r2->type |= ARM_CP_ALIAS;
3665     }
3666
3667     /* Check that raw accesses are either forbidden or handled. Note that
3668      * we can't assert this earlier because the setup of fieldoffset for
3669      * banked registers has to be done first.
3670      */
3671     if (!(r2->type & ARM_CP_NO_RAW)) {
3672         assert(!raw_accessors_invalid(r2));
3673     }
3674
3675     /* Overriding of an existing definition must be explicitly
3676      * requested.
3677      */
3678     if (!(r->type & ARM_CP_OVERRIDE)) {
3679         ARMCPRegInfo *oldreg;
3680         oldreg = g_hash_table_lookup(cpu->cp_regs, key);
3681         if (oldreg && !(oldreg->type & ARM_CP_OVERRIDE)) {
3682             fprintf(stderr, "Register redefined: cp=%d %d bit "
3683                     "crn=%d crm=%d opc1=%d opc2=%d, "
3684                     "was %s, now %s\n", r2->cp, 32 + 32 * is64,
3685                     r2->crn, r2->crm, r2->opc1, r2->opc2,
3686                     oldreg->name, r2->name);
3687             g_assert_not_reached();
3688         }
3689     }
3690     g_hash_table_insert(cpu->cp_regs, key, r2);
3691 }
3692
3693
3694 void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
3695                                        const ARMCPRegInfo *r, void *opaque)
3696 {
3697     /* Define implementations of coprocessor registers.
3698      * We store these in a hashtable because typically
3699      * there are less than 150 registers in a space which
3700      * is 16*16*16*8*8 = 262144 in size.
3701      * Wildcarding is supported for the crm, opc1 and opc2 fields.
3702      * If a register is defined twice then the second definition is
3703      * used, so this can be used to define some generic registers and
3704      * then override them with implementation specific variations.
3705      * At least one of the original and the second definition should
3706      * include ARM_CP_OVERRIDE in its type bits -- this is just a guard
3707      * against accidental use.
3708      *
3709      * The state field defines whether the register is to be
3710      * visible in the AArch32 or AArch64 execution state. If the
3711      * state is set to ARM_CP_STATE_BOTH then we synthesise a
3712      * reginfo structure for the AArch32 view, which sees the lower
3713      * 32 bits of the 64 bit register.
3714      *
3715      * Only registers visible in AArch64 may set r->opc0; opc0 cannot
3716      * be wildcarded. AArch64 registers are always considered to be 64
3717      * bits; the ARM_CP_64BIT* flag applies only to the AArch32 view of
3718      * the register, if any.
3719      */
3720     int crm, opc1, opc2, state;
3721     int crmmin = (r->crm == CP_ANY) ? 0 : r->crm;
3722     int crmmax = (r->crm == CP_ANY) ? 15 : r->crm;
3723     int opc1min = (r->opc1 == CP_ANY) ? 0 : r->opc1;
3724     int opc1max = (r->opc1 == CP_ANY) ? 7 : r->opc1;
3725     int opc2min = (r->opc2 == CP_ANY) ? 0 : r->opc2;
3726     int opc2max = (r->opc2 == CP_ANY) ? 7 : r->opc2;
3727     /* 64 bit registers have only CRm and Opc1 fields */
3728     assert(!((r->type & ARM_CP_64BIT) && (r->opc2 || r->crn)));
3729     /* op0 only exists in the AArch64 encodings */
3730     assert((r->state != ARM_CP_STATE_AA32) || (r->opc0 == 0));
3731     /* AArch64 regs are all 64 bit so ARM_CP_64BIT is meaningless */
3732     assert((r->state != ARM_CP_STATE_AA64) || !(r->type & ARM_CP_64BIT));
3733     /* The AArch64 pseudocode CheckSystemAccess() specifies that op1
3734      * encodes a minimum access level for the register. We roll this
3735      * runtime check into our general permission check code, so check
3736      * here that the reginfo's specified permissions are strict enough
3737      * to encompass the generic architectural permission check.
3738      */
3739     if (r->state != ARM_CP_STATE_AA32) {
3740         int mask = 0;
3741         switch (r->opc1) {
3742         case 0: case 1: case 2:
3743             /* min_EL EL1 */
3744             mask = PL1_RW;
3745             break;
3746         case 3:
3747             /* min_EL EL0 */
3748             mask = PL0_RW;
3749             break;
3750         case 4:
3751             /* min_EL EL2 */
3752             mask = PL2_RW;
3753             break;
3754         case 5:
3755             /* unallocated encoding, so not possible */
3756             assert(false);
3757             break;
3758         case 6:
3759             /* min_EL EL3 */
3760             mask = PL3_RW;
3761             break;
3762         case 7:
3763             /* min_EL EL1, secure mode only (we don't check the latter) */
3764             mask = PL1_RW;
3765             break;
3766         default:
3767             /* broken reginfo with out-of-range opc1 */
3768             assert(false);
3769             break;
3770         }
3771         /* assert our permissions are not too lax (stricter is fine) */
3772         assert((r->access & ~mask) == 0);
3773     }
3774
3775     /* Check that the register definition has enough info to handle
3776      * reads and writes if they are permitted.
3777      */
3778     if (!(r->type & (ARM_CP_SPECIAL|ARM_CP_CONST))) {
3779         if (r->access & PL3_R) {
3780             assert((r->fieldoffset ||
3781                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
3782                    r->readfn);
3783         }
3784         if (r->access & PL3_W) {
3785             assert((r->fieldoffset ||
3786                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
3787                    r->writefn);
3788         }
3789     }
3790     /* Bad type field probably means missing sentinel at end of reg list */
3791     assert(cptype_valid(r->type));
3792     for (crm = crmmin; crm <= crmmax; crm++) {
3793         for (opc1 = opc1min; opc1 <= opc1max; opc1++) {
3794             for (opc2 = opc2min; opc2 <= opc2max; opc2++) {
3795                 for (state = ARM_CP_STATE_AA32;
3796                      state <= ARM_CP_STATE_AA64; state++) {
3797                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
3798                         continue;
3799                     }
3800                     if (state == ARM_CP_STATE_AA32) {
3801                         /* Under AArch32 CP registers can be common
3802                          * (same for secure and non-secure world) or banked.
3803                          */
3804                         switch (r->secure) {
3805                         case ARM_CP_SECSTATE_S:
3806                         case ARM_CP_SECSTATE_NS:
3807                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3808                                                    r->secure, crm, opc1, opc2);
3809                             break;
3810                         default:
3811                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3812                                                    ARM_CP_SECSTATE_S,
3813                                                    crm, opc1, opc2);
3814                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3815                                                    ARM_CP_SECSTATE_NS,
3816                                                    crm, opc1, opc2);
3817                             break;
3818                         }
3819                     } else {
3820                         /* AArch64 registers get mapped to non-secure instance
3821                          * of AArch32 */
3822                         add_cpreg_to_hashtable(cpu, r, opaque, state,
3823                                                ARM_CP_SECSTATE_NS,
3824                                                crm, opc1, opc2);
3825                     }
3826                 }
3827             }
3828         }
3829     }
3830 }
3831
3832 void define_arm_cp_regs_with_opaque(ARMCPU *cpu,
3833                                     const ARMCPRegInfo *regs, void *opaque)
3834 {
3835     /* Define a whole list of registers */
3836     const ARMCPRegInfo *r;
3837     for (r = regs; r->type != ARM_CP_SENTINEL; r++) {
3838         define_one_arm_cp_reg_with_opaque(cpu, r, opaque);
3839     }
3840 }
3841
3842 const ARMCPRegInfo *get_arm_cp_reginfo(GHashTable *cpregs, uint32_t encoded_cp)
3843 {
3844     return g_hash_table_lookup(cpregs, &encoded_cp);
3845 }
3846
3847 void arm_cp_write_ignore(CPUARMState *env, const ARMCPRegInfo *ri,
3848                          uint64_t value)
3849 {
3850     /* Helper coprocessor write function for write-ignore registers */
3851 }
3852
3853 uint64_t arm_cp_read_zero(CPUARMState *env, const ARMCPRegInfo *ri)
3854 {
3855     /* Helper coprocessor write function for read-as-zero registers */
3856     return 0;
3857 }
3858
3859 void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
3860 {
3861     /* Helper coprocessor reset function for do-nothing-on-reset registers */
3862 }
3863
3864 static int bad_mode_switch(CPUARMState *env, int mode)
3865 {
3866     /* Return true if it is not valid for us to switch to
3867      * this CPU mode (ie all the UNPREDICTABLE cases in
3868      * the ARM ARM CPSRWriteByInstr pseudocode).
3869      */
3870     switch (mode) {
3871     case ARM_CPU_MODE_USR:
3872     case ARM_CPU_MODE_SYS:
3873     case ARM_CPU_MODE_SVC:
3874     case ARM_CPU_MODE_ABT:
3875     case ARM_CPU_MODE_UND:
3876     case ARM_CPU_MODE_IRQ:
3877     case ARM_CPU_MODE_FIQ:
3878         return 0;
3879     case ARM_CPU_MODE_MON:
3880         return !arm_is_secure(env);
3881     default:
3882         return 1;
3883     }
3884 }
3885
3886 uint32_t cpsr_read(CPUARMState *env)
3887 {
3888     int ZF;
3889     ZF = (env->ZF == 0);
3890     return env->uncached_cpsr | (env->NF & 0x80000000) | (ZF << 30) |
3891         (env->CF << 29) | ((env->VF & 0x80000000) >> 3) | (env->QF << 27)
3892         | (env->thumb << 5) | ((env->condexec_bits & 3) << 25)
3893         | ((env->condexec_bits & 0xfc) << 8)
3894         | (env->GE << 16) | (env->daif & CPSR_AIF);
3895 }
3896
3897 void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
3898 {
3899     uint32_t changed_daif;
3900
3901     if (mask & CPSR_NZCV) {
3902         env->ZF = (~val) & CPSR_Z;
3903         env->NF = val;
3904         env->CF = (val >> 29) & 1;
3905         env->VF = (val << 3) & 0x80000000;
3906     }
3907     if (mask & CPSR_Q)
3908         env->QF = ((val & CPSR_Q) != 0);
3909     if (mask & CPSR_T)
3910         env->thumb = ((val & CPSR_T) != 0);
3911     if (mask & CPSR_IT_0_1) {
3912         env->condexec_bits &= ~3;
3913         env->condexec_bits |= (val >> 25) & 3;
3914     }
3915     if (mask & CPSR_IT_2_7) {
3916         env->condexec_bits &= 3;
3917         env->condexec_bits |= (val >> 8) & 0xfc;
3918     }
3919     if (mask & CPSR_GE) {
3920         env->GE = (val >> 16) & 0xf;
3921     }
3922
3923     /* In a V7 implementation that includes the security extensions but does
3924      * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
3925      * whether non-secure software is allowed to change the CPSR_F and CPSR_A
3926      * bits respectively.
3927      *
3928      * In a V8 implementation, it is permitted for privileged software to
3929      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
3930      */
3931     if (!arm_feature(env, ARM_FEATURE_V8) &&
3932         arm_feature(env, ARM_FEATURE_EL3) &&
3933         !arm_feature(env, ARM_FEATURE_EL2) &&
3934         !arm_is_secure(env)) {
3935
3936         changed_daif = (env->daif ^ val) & mask;
3937
3938         if (changed_daif & CPSR_A) {
3939             /* Check to see if we are allowed to change the masking of async
3940              * abort exceptions from a non-secure state.
3941              */
3942             if (!(env->cp15.scr_el3 & SCR_AW)) {
3943                 qemu_log_mask(LOG_GUEST_ERROR,
3944                               "Ignoring attempt to switch CPSR_A flag from "
3945                               "non-secure world with SCR.AW bit clear\n");
3946                 mask &= ~CPSR_A;
3947             }
3948         }
3949
3950         if (changed_daif & CPSR_F) {
3951             /* Check to see if we are allowed to change the masking of FIQ
3952              * exceptions from a non-secure state.
3953              */
3954             if (!(env->cp15.scr_el3 & SCR_FW)) {
3955                 qemu_log_mask(LOG_GUEST_ERROR,
3956                               "Ignoring attempt to switch CPSR_F flag from "
3957                               "non-secure world with SCR.FW bit clear\n");
3958                 mask &= ~CPSR_F;
3959             }
3960
3961             /* Check whether non-maskable FIQ (NMFI) support is enabled.
3962              * If this bit is set software is not allowed to mask
3963              * FIQs, but is allowed to set CPSR_F to 0.
3964              */
3965             if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
3966                 (val & CPSR_F)) {
3967                 qemu_log_mask(LOG_GUEST_ERROR,
3968                               "Ignoring attempt to enable CPSR_F flag "
3969                               "(non-maskable FIQ [NMFI] support enabled)\n");
3970                 mask &= ~CPSR_F;
3971             }
3972         }
3973     }
3974
3975     env->daif &= ~(CPSR_AIF & mask);
3976     env->daif |= val & CPSR_AIF & mask;
3977
3978     if ((env->uncached_cpsr ^ val) & mask & CPSR_M) {
3979         if (bad_mode_switch(env, val & CPSR_M)) {
3980             /* Attempt to switch to an invalid mode: this is UNPREDICTABLE.
3981              * We choose to ignore the attempt and leave the CPSR M field
3982              * untouched.
3983              */
3984             mask &= ~CPSR_M;
3985         } else {
3986             switch_mode(env, val & CPSR_M);
3987         }
3988     }
3989     mask &= ~CACHED_CPSR_BITS;
3990     env->uncached_cpsr = (env->uncached_cpsr & ~mask) | (val & mask);
3991 }
3992
3993 /* Sign/zero extend */
3994 uint32_t HELPER(sxtb16)(uint32_t x)
3995 {
3996     uint32_t res;
3997     res = (uint16_t)(int8_t)x;
3998     res |= (uint32_t)(int8_t)(x >> 16) << 16;
3999     return res;
4000 }
4001
4002 uint32_t HELPER(uxtb16)(uint32_t x)
4003 {
4004     uint32_t res;
4005     res = (uint16_t)(uint8_t)x;
4006     res |= (uint32_t)(uint8_t)(x >> 16) << 16;
4007     return res;
4008 }
4009
4010 uint32_t HELPER(clz)(uint32_t x)
4011 {
4012     return clz32(x);
4013 }
4014
4015 int32_t HELPER(sdiv)(int32_t num, int32_t den)
4016 {
4017     if (den == 0)
4018       return 0;
4019     if (num == INT_MIN && den == -1)
4020       return INT_MIN;
4021     return num / den;
4022 }
4023
4024 uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
4025 {
4026     if (den == 0)
4027       return 0;
4028     return num / den;
4029 }
4030
4031 uint32_t HELPER(rbit)(uint32_t x)
4032 {
4033     x =  ((x & 0xff000000) >> 24)
4034        | ((x & 0x00ff0000) >> 8)
4035        | ((x & 0x0000ff00) << 8)
4036        | ((x & 0x000000ff) << 24);
4037     x =  ((x & 0xf0f0f0f0) >> 4)
4038        | ((x & 0x0f0f0f0f) << 4);
4039     x =  ((x & 0x88888888) >> 3)
4040        | ((x & 0x44444444) >> 1)
4041        | ((x & 0x22222222) << 1)
4042        | ((x & 0x11111111) << 3);
4043     return x;
4044 }
4045
4046 #if defined(CONFIG_USER_ONLY)
4047
4048 int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
4049                              int mmu_idx)
4050 {
4051     ARMCPU *cpu = ARM_CPU(cs);
4052     CPUARMState *env = &cpu->env;
4053
4054     env->exception.vaddress = address;
4055     if (rw == 2) {
4056         cs->exception_index = EXCP_PREFETCH_ABORT;
4057     } else {
4058         cs->exception_index = EXCP_DATA_ABORT;
4059     }
4060     return 1;
4061 }
4062
4063 /* These should probably raise undefined insn exceptions.  */
4064 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
4065 {
4066     ARMCPU *cpu = arm_env_get_cpu(env);
4067
4068     cpu_abort(CPU(cpu), "v7m_msr %d\n", reg);
4069 }
4070
4071 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
4072 {
4073     ARMCPU *cpu = arm_env_get_cpu(env);
4074
4075     cpu_abort(CPU(cpu), "v7m_mrs %d\n", reg);
4076     return 0;
4077 }
4078
4079 void switch_mode(CPUARMState *env, int mode)
4080 {
4081     ARMCPU *cpu = arm_env_get_cpu(env);
4082
4083     if (mode != ARM_CPU_MODE_USR) {
4084         cpu_abort(CPU(cpu), "Tried to switch out of user mode\n");
4085     }
4086 }
4087
4088 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
4089 {
4090     ARMCPU *cpu = arm_env_get_cpu(env);
4091
4092     cpu_abort(CPU(cpu), "banked r13 write\n");
4093 }
4094
4095 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
4096 {
4097     ARMCPU *cpu = arm_env_get_cpu(env);
4098
4099     cpu_abort(CPU(cpu), "banked r13 read\n");
4100     return 0;
4101 }
4102
4103 unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
4104 {
4105     return 1;
4106 }
4107
4108 void aarch64_sync_64_to_32(CPUARMState *env)
4109 {
4110     g_assert_not_reached();
4111 }
4112
4113 #else
4114
4115 /* Map CPU modes onto saved register banks.  */
4116 int bank_number(int mode)
4117 {
4118     switch (mode) {
4119     case ARM_CPU_MODE_USR:
4120     case ARM_CPU_MODE_SYS:
4121         return 0;
4122     case ARM_CPU_MODE_SVC:
4123         return 1;
4124     case ARM_CPU_MODE_ABT:
4125         return 2;
4126     case ARM_CPU_MODE_UND:
4127         return 3;
4128     case ARM_CPU_MODE_IRQ:
4129         return 4;
4130     case ARM_CPU_MODE_FIQ:
4131         return 5;
4132     case ARM_CPU_MODE_HYP:
4133         return 6;
4134     case ARM_CPU_MODE_MON:
4135         return 7;
4136     }
4137     hw_error("bank number requested for bad CPSR mode value 0x%x\n", mode);
4138 }
4139
4140 void switch_mode(CPUARMState *env, int mode)
4141 {
4142     int old_mode;
4143     int i;
4144
4145     old_mode = env->uncached_cpsr & CPSR_M;
4146     if (mode == old_mode)
4147         return;
4148
4149     if (old_mode == ARM_CPU_MODE_FIQ) {
4150         memcpy (env->fiq_regs, env->regs + 8, 5 * sizeof(uint32_t));
4151         memcpy (env->regs + 8, env->usr_regs, 5 * sizeof(uint32_t));
4152     } else if (mode == ARM_CPU_MODE_FIQ) {
4153         memcpy (env->usr_regs, env->regs + 8, 5 * sizeof(uint32_t));
4154         memcpy (env->regs + 8, env->fiq_regs, 5 * sizeof(uint32_t));
4155     }
4156
4157     i = bank_number(old_mode);
4158     env->banked_r13[i] = env->regs[13];
4159     env->banked_r14[i] = env->regs[14];
4160     env->banked_spsr[i] = env->spsr;
4161
4162     i = bank_number(mode);
4163     env->regs[13] = env->banked_r13[i];
4164     env->regs[14] = env->banked_r14[i];
4165     env->spsr = env->banked_spsr[i];
4166 }
4167
4168 /* Physical Interrupt Target EL Lookup Table
4169  *
4170  * [ From ARM ARM section G1.13.4 (Table G1-15) ]
4171  *
4172  * The below multi-dimensional table is used for looking up the target
4173  * exception level given numerous condition criteria.  Specifically, the
4174  * target EL is based on SCR and HCR routing controls as well as the
4175  * currently executing EL and secure state.
4176  *
4177  *    Dimensions:
4178  *    target_el_table[2][2][2][2][2][4]
4179  *                    |  |  |  |  |  +--- Current EL
4180  *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
4181  *                    |  |  |  +--------- HCR mask override
4182  *                    |  |  +------------ SCR exec state control
4183  *                    |  +--------------- SCR mask override
4184  *                    +------------------ 32-bit(0)/64-bit(1) EL3
4185  *
4186  *    The table values are as such:
4187  *    0-3 = EL0-EL3
4188  *     -1 = Cannot occur
4189  *
4190  * The ARM ARM target EL table includes entries indicating that an "exception
4191  * is not taken".  The two cases where this is applicable are:
4192  *    1) An exception is taken from EL3 but the SCR does not have the exception
4193  *    routed to EL3.
4194  *    2) An exception is taken from EL2 but the HCR does not have the exception
4195  *    routed to EL2.
4196  * In these two cases, the below table contain a target of EL1.  This value is
4197  * returned as it is expected that the consumer of the table data will check
4198  * for "target EL >= current EL" to ensure the exception is not taken.
4199  *
4200  *            SCR     HCR
4201  *         64  EA     AMO                 From
4202  *        BIT IRQ     IMO      Non-secure         Secure
4203  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
4204  */
4205 const int8_t target_el_table[2][2][2][2][2][4] = {
4206     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
4207        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
4208       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
4209        {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
4210      {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
4211        {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
4212       {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
4213        {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
4214     {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
4215        {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},
4216       {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1, -1,  1 },},
4217        {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},},
4218      {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
4219        {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
4220       {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
4221        {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},},},
4222 };
4223
4224 /*
4225  * Determine the target EL for physical exceptions
4226  */
4227 static inline uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
4228                                         uint32_t cur_el, bool secure)
4229 {
4230     CPUARMState *env = cs->env_ptr;
4231     int rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
4232     int scr;
4233     int hcr;
4234     int target_el;
4235     int is64 = arm_el_is_aa64(env, 3);
4236
4237     switch (excp_idx) {
4238     case EXCP_IRQ:
4239         scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
4240         hcr = ((env->cp15.hcr_el2 & HCR_IMO) == HCR_IMO);
4241         break;
4242     case EXCP_FIQ:
4243         scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
4244         hcr = ((env->cp15.hcr_el2 & HCR_FMO) == HCR_FMO);
4245         break;
4246     default:
4247         scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
4248         hcr = ((env->cp15.hcr_el2 & HCR_AMO) == HCR_AMO);
4249         break;
4250     };
4251
4252     /* If HCR.TGE is set then HCR is treated as being 1 */
4253     hcr |= ((env->cp15.hcr_el2 & HCR_TGE) == HCR_TGE);
4254
4255     /* Perform a table-lookup for the target EL given the current state */
4256     target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
4257
4258     assert(target_el > 0);
4259
4260     return target_el;
4261 }
4262
4263 /*
4264  * Determine the target EL for a given exception type.
4265  */
4266 unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
4267 {
4268     ARMCPU *cpu = ARM_CPU(cs);
4269     CPUARMState *env = &cpu->env;
4270     unsigned int cur_el = arm_current_el(env);
4271     unsigned int target_el;
4272     bool secure = arm_is_secure(env);
4273
4274     switch (excp_idx) {
4275     case EXCP_HVC:
4276     case EXCP_HYP_TRAP:
4277         target_el = 2;
4278         break;
4279     case EXCP_SMC:
4280         target_el = 3;
4281         break;
4282     case EXCP_FIQ:
4283     case EXCP_IRQ:
4284         target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure);
4285         break;
4286     case EXCP_VIRQ:
4287     case EXCP_VFIQ:
4288         target_el = 1;
4289         break;
4290     default:
4291         target_el = MAX(cur_el, 1);
4292         break;
4293     }
4294     return target_el;
4295 }
4296
4297 static void v7m_push(CPUARMState *env, uint32_t val)
4298 {
4299     CPUState *cs = CPU(arm_env_get_cpu(env));
4300
4301     env->regs[13] -= 4;
4302     stl_phys(cs->as, env->regs[13], val);
4303 }
4304
4305 static uint32_t v7m_pop(CPUARMState *env)
4306 {
4307     CPUState *cs = CPU(arm_env_get_cpu(env));
4308     uint32_t val;
4309
4310     val = ldl_phys(cs->as, env->regs[13]);
4311     env->regs[13] += 4;
4312     return val;
4313 }
4314
4315 /* Switch to V7M main or process stack pointer.  */
4316 static void switch_v7m_sp(CPUARMState *env, int process)
4317 {
4318     uint32_t tmp;
4319     if (env->v7m.current_sp != process) {
4320         tmp = env->v7m.other_sp;
4321         env->v7m.other_sp = env->regs[13];
4322         env->regs[13] = tmp;
4323         env->v7m.current_sp = process;
4324     }
4325 }
4326
4327 static void do_v7m_exception_exit(CPUARMState *env)
4328 {
4329     uint32_t type;
4330     uint32_t xpsr;
4331
4332     type = env->regs[15];
4333     if (env->v7m.exception != 0)
4334         armv7m_nvic_complete_irq(env->nvic, env->v7m.exception);
4335
4336     /* Switch to the target stack.  */
4337     switch_v7m_sp(env, (type & 4) != 0);
4338     /* Pop registers.  */
4339     env->regs[0] = v7m_pop(env);
4340     env->regs[1] = v7m_pop(env);
4341     env->regs[2] = v7m_pop(env);
4342     env->regs[3] = v7m_pop(env);
4343     env->regs[12] = v7m_pop(env);
4344     env->regs[14] = v7m_pop(env);
4345     env->regs[15] = v7m_pop(env);
4346     if (env->regs[15] & 1) {
4347         qemu_log_mask(LOG_GUEST_ERROR,
4348                       "M profile return from interrupt with misaligned "
4349                       "PC is UNPREDICTABLE\n");
4350         /* Actual hardware seems to ignore the lsbit, and there are several
4351          * RTOSes out there which incorrectly assume the r15 in the stack
4352          * frame should be a Thumb-style "lsbit indicates ARM/Thumb" value.
4353          */
4354         env->regs[15] &= ~1U;
4355     }
4356     xpsr = v7m_pop(env);
4357     xpsr_write(env, xpsr, 0xfffffdff);
4358     /* Undo stack alignment.  */
4359     if (xpsr & 0x200)
4360         env->regs[13] |= 4;
4361     /* ??? The exception return type specifies Thread/Handler mode.  However
4362        this is also implied by the xPSR value. Not sure what to do
4363        if there is a mismatch.  */
4364     /* ??? Likewise for mismatches between the CONTROL register and the stack
4365        pointer.  */
4366 }
4367
4368 void arm_v7m_cpu_do_interrupt(CPUState *cs)
4369 {
4370     ARMCPU *cpu = ARM_CPU(cs);
4371     CPUARMState *env = &cpu->env;
4372     uint32_t xpsr = xpsr_read(env);
4373     uint32_t lr;
4374     uint32_t addr;
4375
4376     arm_log_exception(cs->exception_index);
4377
4378     lr = 0xfffffff1;
4379     if (env->v7m.current_sp)
4380         lr |= 4;
4381     if (env->v7m.exception == 0)
4382         lr |= 8;
4383
4384     /* For exceptions we just mark as pending on the NVIC, and let that
4385        handle it.  */
4386     /* TODO: Need to escalate if the current priority is higher than the
4387        one we're raising.  */
4388     switch (cs->exception_index) {
4389     case EXCP_UDEF:
4390         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE);
4391         return;
4392     case EXCP_SWI:
4393         /* The PC already points to the next instruction.  */
4394         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SVC);
4395         return;
4396     case EXCP_PREFETCH_ABORT:
4397     case EXCP_DATA_ABORT:
4398         /* TODO: if we implemented the MPU registers, this is where we
4399          * should set the MMFAR, etc from exception.fsr and exception.vaddress.
4400          */
4401         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_MEM);
4402         return;
4403     case EXCP_BKPT:
4404         if (semihosting_enabled) {
4405             int nr;
4406             nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
4407             if (nr == 0xab) {
4408                 env->regs[15] += 2;
4409                 env->regs[0] = do_arm_semihosting(env);
4410                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4411                 return;
4412             }
4413         }
4414         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_DEBUG);
4415         return;
4416     case EXCP_IRQ:
4417         env->v7m.exception = armv7m_nvic_acknowledge_irq(env->nvic);
4418         break;
4419     case EXCP_EXCEPTION_EXIT:
4420         do_v7m_exception_exit(env);
4421         return;
4422     default:
4423         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
4424         return; /* Never happens.  Keep compiler happy.  */
4425     }
4426
4427     /* Align stack pointer.  */
4428     /* ??? Should only do this if Configuration Control Register
4429        STACKALIGN bit is set.  */
4430     if (env->regs[13] & 4) {
4431         env->regs[13] -= 4;
4432         xpsr |= 0x200;
4433     }
4434     /* Switch to the handler mode.  */
4435     v7m_push(env, xpsr);
4436     v7m_push(env, env->regs[15]);
4437     v7m_push(env, env->regs[14]);
4438     v7m_push(env, env->regs[12]);
4439     v7m_push(env, env->regs[3]);
4440     v7m_push(env, env->regs[2]);
4441     v7m_push(env, env->regs[1]);
4442     v7m_push(env, env->regs[0]);
4443     switch_v7m_sp(env, 0);
4444     /* Clear IT bits */
4445     env->condexec_bits = 0;
4446     env->regs[14] = lr;
4447     addr = ldl_phys(cs->as, env->v7m.vecbase + env->v7m.exception * 4);
4448     env->regs[15] = addr & 0xfffffffe;
4449     env->thumb = addr & 1;
4450 }
4451
4452 /* Function used to synchronize QEMU's AArch64 register set with AArch32
4453  * register set.  This is necessary when switching between AArch32 and AArch64
4454  * execution state.
4455  */
4456 void aarch64_sync_32_to_64(CPUARMState *env)
4457 {
4458     int i;
4459     uint32_t mode = env->uncached_cpsr & CPSR_M;
4460
4461     /* We can blanket copy R[0:7] to X[0:7] */
4462     for (i = 0; i < 8; i++) {
4463         env->xregs[i] = env->regs[i];
4464     }
4465
4466     /* Unless we are in FIQ mode, x8-x12 come from the user registers r8-r12.
4467      * Otherwise, they come from the banked user regs.
4468      */
4469     if (mode == ARM_CPU_MODE_FIQ) {
4470         for (i = 8; i < 13; i++) {
4471             env->xregs[i] = env->usr_regs[i - 8];
4472         }
4473     } else {
4474         for (i = 8; i < 13; i++) {
4475             env->xregs[i] = env->regs[i];
4476         }
4477     }
4478
4479     /* Registers x13-x23 are the various mode SP and FP registers. Registers
4480      * r13 and r14 are only copied if we are in that mode, otherwise we copy
4481      * from the mode banked register.
4482      */
4483     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
4484         env->xregs[13] = env->regs[13];
4485         env->xregs[14] = env->regs[14];
4486     } else {
4487         env->xregs[13] = env->banked_r13[bank_number(ARM_CPU_MODE_USR)];
4488         /* HYP is an exception in that it is copied from r14 */
4489         if (mode == ARM_CPU_MODE_HYP) {
4490             env->xregs[14] = env->regs[14];
4491         } else {
4492             env->xregs[14] = env->banked_r14[bank_number(ARM_CPU_MODE_USR)];
4493         }
4494     }
4495
4496     if (mode == ARM_CPU_MODE_HYP) {
4497         env->xregs[15] = env->regs[13];
4498     } else {
4499         env->xregs[15] = env->banked_r13[bank_number(ARM_CPU_MODE_HYP)];
4500     }
4501
4502     if (mode == ARM_CPU_MODE_IRQ) {
4503         env->xregs[16] = env->regs[13];
4504         env->xregs[17] = env->regs[14];
4505     } else {
4506         env->xregs[16] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
4507         env->xregs[17] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
4508     }
4509
4510     if (mode == ARM_CPU_MODE_SVC) {
4511         env->xregs[18] = env->regs[13];
4512         env->xregs[19] = env->regs[14];
4513     } else {
4514         env->xregs[18] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
4515         env->xregs[19] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
4516     }
4517
4518     if (mode == ARM_CPU_MODE_ABT) {
4519         env->xregs[20] = env->regs[13];
4520         env->xregs[21] = env->regs[14];
4521     } else {
4522         env->xregs[20] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
4523         env->xregs[21] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
4524     }
4525
4526     if (mode == ARM_CPU_MODE_UND) {
4527         env->xregs[22] = env->regs[13];
4528         env->xregs[23] = env->regs[14];
4529     } else {
4530         env->xregs[22] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
4531         env->xregs[23] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
4532     }
4533
4534     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
4535      * mode, then we can copy from r8-r14.  Otherwise, we copy from the
4536      * FIQ bank for r8-r14.
4537      */
4538     if (mode == ARM_CPU_MODE_FIQ) {
4539         for (i = 24; i < 31; i++) {
4540             env->xregs[i] = env->regs[i - 16];   /* X[24:30] <- R[8:14] */
4541         }
4542     } else {
4543         for (i = 24; i < 29; i++) {
4544             env->xregs[i] = env->fiq_regs[i - 24];
4545         }
4546         env->xregs[29] = env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)];
4547         env->xregs[30] = env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)];
4548     }
4549
4550     env->pc = env->regs[15];
4551 }
4552
4553 /* Function used to synchronize QEMU's AArch32 register set with AArch64
4554  * register set.  This is necessary when switching between AArch32 and AArch64
4555  * execution state.
4556  */
4557 void aarch64_sync_64_to_32(CPUARMState *env)
4558 {
4559     int i;
4560     uint32_t mode = env->uncached_cpsr & CPSR_M;
4561
4562     /* We can blanket copy X[0:7] to R[0:7] */
4563     for (i = 0; i < 8; i++) {
4564         env->regs[i] = env->xregs[i];
4565     }
4566
4567     /* Unless we are in FIQ mode, r8-r12 come from the user registers x8-x12.
4568      * Otherwise, we copy x8-x12 into the banked user regs.
4569      */
4570     if (mode == ARM_CPU_MODE_FIQ) {
4571         for (i = 8; i < 13; i++) {
4572             env->usr_regs[i - 8] = env->xregs[i];
4573         }
4574     } else {
4575         for (i = 8; i < 13; i++) {
4576             env->regs[i] = env->xregs[i];
4577         }
4578     }
4579
4580     /* Registers r13 & r14 depend on the current mode.
4581      * If we are in a given mode, we copy the corresponding x registers to r13
4582      * and r14.  Otherwise, we copy the x register to the banked r13 and r14
4583      * for the mode.
4584      */
4585     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
4586         env->regs[13] = env->xregs[13];
4587         env->regs[14] = env->xregs[14];
4588     } else {
4589         env->banked_r13[bank_number(ARM_CPU_MODE_USR)] = env->xregs[13];
4590
4591         /* HYP is an exception in that it does not have its own banked r14 but
4592          * shares the USR r14
4593          */
4594         if (mode == ARM_CPU_MODE_HYP) {
4595             env->regs[14] = env->xregs[14];
4596         } else {
4597             env->banked_r14[bank_number(ARM_CPU_MODE_USR)] = env->xregs[14];
4598         }
4599     }
4600
4601     if (mode == ARM_CPU_MODE_HYP) {
4602         env->regs[13] = env->xregs[15];
4603     } else {
4604         env->banked_r13[bank_number(ARM_CPU_MODE_HYP)] = env->xregs[15];
4605     }
4606
4607     if (mode == ARM_CPU_MODE_IRQ) {
4608         env->regs[13] = env->xregs[16];
4609         env->regs[14] = env->xregs[17];
4610     } else {
4611         env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
4612         env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
4613     }
4614
4615     if (mode == ARM_CPU_MODE_SVC) {
4616         env->regs[13] = env->xregs[18];
4617         env->regs[14] = env->xregs[19];
4618     } else {
4619         env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
4620         env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
4621     }
4622
4623     if (mode == ARM_CPU_MODE_ABT) {
4624         env->regs[13] = env->xregs[20];
4625         env->regs[14] = env->xregs[21];
4626     } else {
4627         env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
4628         env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
4629     }
4630
4631     if (mode == ARM_CPU_MODE_UND) {
4632         env->regs[13] = env->xregs[22];
4633         env->regs[14] = env->xregs[23];
4634     } else {
4635         env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
4636         env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
4637     }
4638
4639     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
4640      * mode, then we can copy to r8-r14.  Otherwise, we copy to the
4641      * FIQ bank for r8-r14.
4642      */
4643     if (mode == ARM_CPU_MODE_FIQ) {
4644         for (i = 24; i < 31; i++) {
4645             env->regs[i - 16] = env->xregs[i];   /* X[24:30] -> R[8:14] */
4646         }
4647     } else {
4648         for (i = 24; i < 29; i++) {
4649             env->fiq_regs[i - 24] = env->xregs[i];
4650         }
4651         env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[29];
4652         env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[30];
4653     }
4654
4655     env->regs[15] = env->pc;
4656 }
4657
4658 /* Handle a CPU exception.  */
4659 void arm_cpu_do_interrupt(CPUState *cs)
4660 {
4661     ARMCPU *cpu = ARM_CPU(cs);
4662     CPUARMState *env = &cpu->env;
4663     uint32_t addr;
4664     uint32_t mask;
4665     int new_mode;
4666     uint32_t offset;
4667     uint32_t moe;
4668
4669     assert(!IS_M(env));
4670
4671     arm_log_exception(cs->exception_index);
4672
4673     if (arm_is_psci_call(cpu, cs->exception_index)) {
4674         arm_handle_psci_call(cpu);
4675         qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
4676         return;
4677     }
4678
4679     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
4680     switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
4681     case EC_BREAKPOINT:
4682     case EC_BREAKPOINT_SAME_EL:
4683         moe = 1;
4684         break;
4685     case EC_WATCHPOINT:
4686     case EC_WATCHPOINT_SAME_EL:
4687         moe = 10;
4688         break;
4689     case EC_AA32_BKPT:
4690         moe = 3;
4691         break;
4692     case EC_VECTORCATCH:
4693         moe = 5;
4694         break;
4695     default:
4696         moe = 0;
4697         break;
4698     }
4699
4700     if (moe) {
4701         env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
4702     }
4703
4704     /* TODO: Vectored interrupt controller.  */
4705     switch (cs->exception_index) {
4706     case EXCP_UDEF:
4707         new_mode = ARM_CPU_MODE_UND;
4708         addr = 0x04;
4709         mask = CPSR_I;
4710         if (env->thumb)
4711             offset = 2;
4712         else
4713             offset = 4;
4714         break;
4715     case EXCP_SWI:
4716         if (semihosting_enabled) {
4717             /* Check for semihosting interrupt.  */
4718             if (env->thumb) {
4719                 mask = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
4720                     & 0xff;
4721             } else {
4722                 mask = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
4723                     & 0xffffff;
4724             }
4725             /* Only intercept calls from privileged modes, to provide some
4726                semblance of security.  */
4727             if (((mask == 0x123456 && !env->thumb)
4728                     || (mask == 0xab && env->thumb))
4729                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
4730                 env->regs[0] = do_arm_semihosting(env);
4731                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4732                 return;
4733             }
4734         }
4735         new_mode = ARM_CPU_MODE_SVC;
4736         addr = 0x08;
4737         mask = CPSR_I;
4738         /* The PC already points to the next instruction.  */
4739         offset = 0;
4740         break;
4741     case EXCP_BKPT:
4742         /* See if this is a semihosting syscall.  */
4743         if (env->thumb && semihosting_enabled) {
4744             mask = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
4745             if (mask == 0xab
4746                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
4747                 env->regs[15] += 2;
4748                 env->regs[0] = do_arm_semihosting(env);
4749                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4750                 return;
4751             }
4752         }
4753         env->exception.fsr = 2;
4754         /* Fall through to prefetch abort.  */
4755     case EXCP_PREFETCH_ABORT:
4756         A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
4757         A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
4758         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
4759                       env->exception.fsr, (uint32_t)env->exception.vaddress);
4760         new_mode = ARM_CPU_MODE_ABT;
4761         addr = 0x0c;
4762         mask = CPSR_A | CPSR_I;
4763         offset = 4;
4764         break;
4765     case EXCP_DATA_ABORT:
4766         A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
4767         A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
4768         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
4769                       env->exception.fsr,
4770                       (uint32_t)env->exception.vaddress);
4771         new_mode = ARM_CPU_MODE_ABT;
4772         addr = 0x10;
4773         mask = CPSR_A | CPSR_I;
4774         offset = 8;
4775         break;
4776     case EXCP_IRQ:
4777         new_mode = ARM_CPU_MODE_IRQ;
4778         addr = 0x18;
4779         /* Disable IRQ and imprecise data aborts.  */
4780         mask = CPSR_A | CPSR_I;
4781         offset = 4;
4782         if (env->cp15.scr_el3 & SCR_IRQ) {
4783             /* IRQ routed to monitor mode */
4784             new_mode = ARM_CPU_MODE_MON;
4785             mask |= CPSR_F;
4786         }
4787         break;
4788     case EXCP_FIQ:
4789         new_mode = ARM_CPU_MODE_FIQ;
4790         addr = 0x1c;
4791         /* Disable FIQ, IRQ and imprecise data aborts.  */
4792         mask = CPSR_A | CPSR_I | CPSR_F;
4793         if (env->cp15.scr_el3 & SCR_FIQ) {
4794             /* FIQ routed to monitor mode */
4795             new_mode = ARM_CPU_MODE_MON;
4796         }
4797         offset = 4;
4798         break;
4799     case EXCP_SMC:
4800         new_mode = ARM_CPU_MODE_MON;
4801         addr = 0x08;
4802         mask = CPSR_A | CPSR_I | CPSR_F;
4803         offset = 0;
4804         break;
4805     default:
4806         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
4807         return; /* Never happens.  Keep compiler happy.  */
4808     }
4809
4810     if (new_mode == ARM_CPU_MODE_MON) {
4811         addr += env->cp15.mvbar;
4812     } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
4813         /* High vectors. When enabled, base address cannot be remapped. */
4814         addr += 0xffff0000;
4815     } else {
4816         /* ARM v7 architectures provide a vector base address register to remap
4817          * the interrupt vector table.
4818          * This register is only followed in non-monitor mode, and is banked.
4819          * Note: only bits 31:5 are valid.
4820          */
4821         addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
4822     }
4823
4824     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
4825         env->cp15.scr_el3 &= ~SCR_NS;
4826     }
4827
4828     switch_mode (env, new_mode);
4829     /* For exceptions taken to AArch32 we must clear the SS bit in both
4830      * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
4831      */
4832     env->uncached_cpsr &= ~PSTATE_SS;
4833     env->spsr = cpsr_read(env);
4834     /* Clear IT bits.  */
4835     env->condexec_bits = 0;
4836     /* Switch to the new mode, and to the correct instruction set.  */
4837     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
4838     env->daif |= mask;
4839     /* this is a lie, as the was no c1_sys on V4T/V5, but who cares
4840      * and we should just guard the thumb mode on V4 */
4841     if (arm_feature(env, ARM_FEATURE_V4T)) {
4842         env->thumb = (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
4843     }
4844     env->regs[14] = env->regs[15] + offset;
4845     env->regs[15] = addr;
4846     cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
4847 }
4848
4849
4850 /* Return the exception level which controls this address translation regime */
4851 static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx)
4852 {
4853     switch (mmu_idx) {
4854     case ARMMMUIdx_S2NS:
4855     case ARMMMUIdx_S1E2:
4856         return 2;
4857     case ARMMMUIdx_S1E3:
4858         return 3;
4859     case ARMMMUIdx_S1SE0:
4860         return arm_el_is_aa64(env, 3) ? 1 : 3;
4861     case ARMMMUIdx_S1SE1:
4862     case ARMMMUIdx_S1NSE0:
4863     case ARMMMUIdx_S1NSE1:
4864         return 1;
4865     default:
4866         g_assert_not_reached();
4867     }
4868 }
4869
4870 /* Return true if this address translation regime is secure */
4871 static inline bool regime_is_secure(CPUARMState *env, ARMMMUIdx mmu_idx)
4872 {
4873     switch (mmu_idx) {
4874     case ARMMMUIdx_S12NSE0:
4875     case ARMMMUIdx_S12NSE1:
4876     case ARMMMUIdx_S1NSE0:
4877     case ARMMMUIdx_S1NSE1:
4878     case ARMMMUIdx_S1E2:
4879     case ARMMMUIdx_S2NS:
4880         return false;
4881     case ARMMMUIdx_S1E3:
4882     case ARMMMUIdx_S1SE0:
4883     case ARMMMUIdx_S1SE1:
4884         return true;
4885     default:
4886         g_assert_not_reached();
4887     }
4888 }
4889
4890 /* Return the SCTLR value which controls this address translation regime */
4891 static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx)
4892 {
4893     return env->cp15.sctlr_el[regime_el(env, mmu_idx)];
4894 }
4895
4896 /* Return true if the specified stage of address translation is disabled */
4897 static inline bool regime_translation_disabled(CPUARMState *env,
4898                                                ARMMMUIdx mmu_idx)
4899 {
4900     if (mmu_idx == ARMMMUIdx_S2NS) {
4901         return (env->cp15.hcr_el2 & HCR_VM) == 0;
4902     }
4903     return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
4904 }
4905
4906 /* Return the TCR controlling this translation regime */
4907 static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx)
4908 {
4909     if (mmu_idx == ARMMMUIdx_S2NS) {
4910         /* TODO: return VTCR_EL2 */
4911         g_assert_not_reached();
4912     }
4913     return &env->cp15.tcr_el[regime_el(env, mmu_idx)];
4914 }
4915
4916 /* Return true if the translation regime is using LPAE format page tables */
4917 static inline bool regime_using_lpae_format(CPUARMState *env,
4918                                             ARMMMUIdx mmu_idx)
4919 {
4920     int el = regime_el(env, mmu_idx);
4921     if (el == 2 || arm_el_is_aa64(env, el)) {
4922         return true;
4923     }
4924     if (arm_feature(env, ARM_FEATURE_LPAE)
4925         && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) {
4926         return true;
4927     }
4928     return false;
4929 }
4930
4931 static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx)
4932 {
4933     switch (mmu_idx) {
4934     case ARMMMUIdx_S1SE0:
4935     case ARMMMUIdx_S1NSE0:
4936         return true;
4937     default:
4938         return false;
4939     case ARMMMUIdx_S12NSE0:
4940     case ARMMMUIdx_S12NSE1:
4941         g_assert_not_reached();
4942     }
4943 }
4944
4945 /* Translate section/page access permissions to page
4946  * R/W protection flags
4947  *
4948  * @env:         CPUARMState
4949  * @mmu_idx:     MMU index indicating required translation regime
4950  * @ap:          The 3-bit access permissions (AP[2:0])
4951  * @domain_prot: The 2-bit domain access permissions
4952  */
4953 static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
4954                                 int ap, int domain_prot)
4955 {
4956     bool is_user = regime_is_user(env, mmu_idx);
4957
4958     if (domain_prot == 3) {
4959         return PAGE_READ | PAGE_WRITE;
4960     }
4961
4962     switch (ap) {
4963     case 0:
4964         if (arm_feature(env, ARM_FEATURE_V7)) {
4965             return 0;
4966         }
4967         switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) {
4968         case SCTLR_S:
4969             return is_user ? 0 : PAGE_READ;
4970         case SCTLR_R:
4971             return PAGE_READ;
4972         default:
4973             return 0;
4974         }
4975     case 1:
4976         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
4977     case 2:
4978         if (is_user) {
4979             return PAGE_READ;
4980         } else {
4981             return PAGE_READ | PAGE_WRITE;
4982         }
4983     case 3:
4984         return PAGE_READ | PAGE_WRITE;
4985     case 4: /* Reserved.  */
4986         return 0;
4987     case 5:
4988         return is_user ? 0 : PAGE_READ;
4989     case 6:
4990         return PAGE_READ;
4991     case 7:
4992         if (!arm_feature(env, ARM_FEATURE_V6K)) {
4993             return 0;
4994         }
4995         return PAGE_READ;
4996     default:
4997         g_assert_not_reached();
4998     }
4999 }
5000
5001 /* Translate section/page access permissions to page
5002  * R/W protection flags.
5003  *
5004  * @ap:      The 2-bit simple AP (AP[2:1])
5005  * @is_user: TRUE if accessing from PL0
5006  */
5007 static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
5008 {
5009     switch (ap) {
5010     case 0:
5011         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
5012     case 1:
5013         return PAGE_READ | PAGE_WRITE;
5014     case 2:
5015         return is_user ? 0 : PAGE_READ;
5016     case 3:
5017         return PAGE_READ;
5018     default:
5019         g_assert_not_reached();
5020     }
5021 }
5022
5023 static inline int
5024 simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
5025 {
5026     return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
5027 }
5028
5029 /* Translate section/page access permissions to protection flags
5030  *
5031  * @env:     CPUARMState
5032  * @mmu_idx: MMU index indicating required translation regime
5033  * @is_aa64: TRUE if AArch64
5034  * @ap:      The 2-bit simple AP (AP[2:1])
5035  * @ns:      NS (non-secure) bit
5036  * @xn:      XN (execute-never) bit
5037  * @pxn:     PXN (privileged execute-never) bit
5038  */
5039 static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
5040                       int ap, int ns, int xn, int pxn)
5041 {
5042     bool is_user = regime_is_user(env, mmu_idx);
5043     int prot_rw, user_rw;
5044     bool have_wxn;
5045     int wxn = 0;
5046
5047     assert(mmu_idx != ARMMMUIdx_S2NS);
5048
5049     user_rw = simple_ap_to_rw_prot_is_user(ap, true);
5050     if (is_user) {
5051         prot_rw = user_rw;
5052     } else {
5053         prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
5054     }
5055
5056     if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) {
5057         return prot_rw;
5058     }
5059
5060     /* TODO have_wxn should be replaced with
5061      *   ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
5062      * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
5063      * compatible processors have EL2, which is required for [U]WXN.
5064      */
5065     have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
5066
5067     if (have_wxn) {
5068         wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
5069     }
5070
5071     if (is_aa64) {
5072         switch (regime_el(env, mmu_idx)) {
5073         case 1:
5074             if (!is_user) {
5075                 xn = pxn || (user_rw & PAGE_WRITE);
5076             }
5077             break;
5078         case 2:
5079         case 3:
5080             break;
5081         }
5082     } else if (arm_feature(env, ARM_FEATURE_V7)) {
5083         switch (regime_el(env, mmu_idx)) {
5084         case 1:
5085         case 3:
5086             if (is_user) {
5087                 xn = xn || !(user_rw & PAGE_READ);
5088             } else {
5089                 int uwxn = 0;
5090                 if (have_wxn) {
5091                     uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
5092                 }
5093                 xn = xn || !(prot_rw & PAGE_READ) || pxn ||
5094                      (uwxn && (user_rw & PAGE_WRITE));
5095             }
5096             break;
5097         case 2:
5098             break;
5099         }
5100     } else {
5101         xn = wxn = 0;
5102     }
5103
5104     if (xn || (wxn && (prot_rw & PAGE_WRITE))) {
5105         return prot_rw;
5106     }
5107     return prot_rw | PAGE_EXEC;
5108 }
5109
5110 static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
5111                                      uint32_t *table, uint32_t address)
5112 {
5113     /* Note that we can only get here for an AArch32 PL0/PL1 lookup */
5114     int el = regime_el(env, mmu_idx);
5115     TCR *tcr = regime_tcr(env, mmu_idx);
5116
5117     if (address & tcr->mask) {
5118         if (tcr->raw_tcr & TTBCR_PD1) {
5119             /* Translation table walk disabled for TTBR1 */
5120             return false;
5121         }
5122         *table = env->cp15.ttbr1_el[el] & 0xffffc000;
5123     } else {
5124         if (tcr->raw_tcr & TTBCR_PD0) {
5125             /* Translation table walk disabled for TTBR0 */
5126             return false;
5127         }
5128         *table = env->cp15.ttbr0_el[el] & tcr->base_mask;
5129     }
5130     *table |= (address >> 18) & 0x3ffc;
5131     return true;
5132 }
5133
5134 /* All loads done in the course of a page table walk go through here.
5135  * TODO: rather than ignoring errors from physical memory reads (which
5136  * are external aborts in ARM terminology) we should propagate this
5137  * error out so that we can turn it into a Data Abort if this walk
5138  * was being done for a CPU load/store or an address translation instruction
5139  * (but not if it was for a debug access).
5140  */
5141 static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure)
5142 {
5143     MemTxAttrs attrs = {};
5144
5145     attrs.secure = is_secure;
5146     return address_space_ldl(cs->as, addr, attrs, NULL);
5147 }
5148
5149 static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure)
5150 {
5151     MemTxAttrs attrs = {};
5152
5153     attrs.secure = is_secure;
5154     return address_space_ldq(cs->as, addr, attrs, NULL);
5155 }
5156
5157 static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
5158                             ARMMMUIdx mmu_idx, hwaddr *phys_ptr,
5159                             int *prot, target_ulong *page_size)
5160 {
5161     CPUState *cs = CPU(arm_env_get_cpu(env));
5162     int code;
5163     uint32_t table;
5164     uint32_t desc;
5165     int type;
5166     int ap;
5167     int domain = 0;
5168     int domain_prot;
5169     hwaddr phys_addr;
5170     uint32_t dacr;
5171
5172     /* Pagetable walk.  */
5173     /* Lookup l1 descriptor.  */
5174     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
5175         /* Section translation fault if page walk is disabled by PD0 or PD1 */
5176         code = 5;
5177         goto do_fault;
5178     }
5179     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5180     type = (desc & 3);
5181     domain = (desc >> 5) & 0x0f;
5182     if (regime_el(env, mmu_idx) == 1) {
5183         dacr = env->cp15.dacr_ns;
5184     } else {
5185         dacr = env->cp15.dacr_s;
5186     }
5187     domain_prot = (dacr >> (domain * 2)) & 3;
5188     if (type == 0) {
5189         /* Section translation fault.  */
5190         code = 5;
5191         goto do_fault;
5192     }
5193     if (domain_prot == 0 || domain_prot == 2) {
5194         if (type == 2)
5195             code = 9; /* Section domain fault.  */
5196         else
5197             code = 11; /* Page domain fault.  */
5198         goto do_fault;
5199     }
5200     if (type == 2) {
5201         /* 1Mb section.  */
5202         phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
5203         ap = (desc >> 10) & 3;
5204         code = 13;
5205         *page_size = 1024 * 1024;
5206     } else {
5207         /* Lookup l2 entry.  */
5208         if (type == 1) {
5209             /* Coarse pagetable.  */
5210             table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
5211         } else {
5212             /* Fine pagetable.  */
5213             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
5214         }
5215         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5216         switch (desc & 3) {
5217         case 0: /* Page translation fault.  */
5218             code = 7;
5219             goto do_fault;
5220         case 1: /* 64k page.  */
5221             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
5222             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
5223             *page_size = 0x10000;
5224             break;
5225         case 2: /* 4k page.  */
5226             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
5227             ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
5228             *page_size = 0x1000;
5229             break;
5230         case 3: /* 1k page.  */
5231             if (type == 1) {
5232                 if (arm_feature(env, ARM_FEATURE_XSCALE)) {
5233                     phys_addr = (desc & 0xfffff000) | (address & 0xfff);
5234                 } else {
5235                     /* Page translation fault.  */
5236                     code = 7;
5237                     goto do_fault;
5238                 }
5239             } else {
5240                 phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
5241             }
5242             ap = (desc >> 4) & 3;
5243             *page_size = 0x400;
5244             break;
5245         default:
5246             /* Never happens, but compiler isn't smart enough to tell.  */
5247             abort();
5248         }
5249         code = 15;
5250     }
5251     *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
5252     *prot |= *prot ? PAGE_EXEC : 0;
5253     if (!(*prot & (1 << access_type))) {
5254         /* Access permission fault.  */
5255         goto do_fault;
5256     }
5257     *phys_ptr = phys_addr;
5258     return 0;
5259 do_fault:
5260     return code | (domain << 4);
5261 }
5262
5263 static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
5264                             ARMMMUIdx mmu_idx, hwaddr *phys_ptr,
5265                             MemTxAttrs *attrs,
5266                             int *prot, target_ulong *page_size)
5267 {
5268     CPUState *cs = CPU(arm_env_get_cpu(env));
5269     int code;
5270     uint32_t table;
5271     uint32_t desc;
5272     uint32_t xn;
5273     uint32_t pxn = 0;
5274     int type;
5275     int ap;
5276     int domain = 0;
5277     int domain_prot;
5278     hwaddr phys_addr;
5279     uint32_t dacr;
5280     bool ns;
5281
5282     /* Pagetable walk.  */
5283     /* Lookup l1 descriptor.  */
5284     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
5285         /* Section translation fault if page walk is disabled by PD0 or PD1 */
5286         code = 5;
5287         goto do_fault;
5288     }
5289     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5290     type = (desc & 3);
5291     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
5292         /* Section translation fault, or attempt to use the encoding
5293          * which is Reserved on implementations without PXN.
5294          */
5295         code = 5;
5296         goto do_fault;
5297     }
5298     if ((type == 1) || !(desc & (1 << 18))) {
5299         /* Page or Section.  */
5300         domain = (desc >> 5) & 0x0f;
5301     }
5302     if (regime_el(env, mmu_idx) == 1) {
5303         dacr = env->cp15.dacr_ns;
5304     } else {
5305         dacr = env->cp15.dacr_s;
5306     }
5307     domain_prot = (dacr >> (domain * 2)) & 3;
5308     if (domain_prot == 0 || domain_prot == 2) {
5309         if (type != 1) {
5310             code = 9; /* Section domain fault.  */
5311         } else {
5312             code = 11; /* Page domain fault.  */
5313         }
5314         goto do_fault;
5315     }
5316     if (type != 1) {
5317         if (desc & (1 << 18)) {
5318             /* Supersection.  */
5319             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
5320             *page_size = 0x1000000;
5321         } else {
5322             /* Section.  */
5323             phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
5324             *page_size = 0x100000;
5325         }
5326         ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
5327         xn = desc & (1 << 4);
5328         pxn = desc & 1;
5329         code = 13;
5330         ns = extract32(desc, 19, 1);
5331     } else {
5332         if (arm_feature(env, ARM_FEATURE_PXN)) {
5333             pxn = (desc >> 2) & 1;
5334         }
5335         ns = extract32(desc, 3, 1);
5336         /* Lookup l2 entry.  */
5337         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
5338         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx));
5339         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
5340         switch (desc & 3) {
5341         case 0: /* Page translation fault.  */
5342             code = 7;
5343             goto do_fault;
5344         case 1: /* 64k page.  */
5345             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
5346             xn = desc & (1 << 15);
5347             *page_size = 0x10000;
5348             break;
5349         case 2: case 3: /* 4k page.  */
5350             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
5351             xn = desc & 1;
5352             *page_size = 0x1000;
5353             break;
5354         default:
5355             /* Never happens, but compiler isn't smart enough to tell.  */
5356             abort();
5357         }
5358         code = 15;
5359     }
5360     if (domain_prot == 3) {
5361         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
5362     } else {
5363         if (pxn && !regime_is_user(env, mmu_idx)) {
5364             xn = 1;
5365         }
5366         if (xn && access_type == 2)
5367             goto do_fault;
5368
5369         if (arm_feature(env, ARM_FEATURE_V6K) &&
5370                 (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
5371             /* The simplified model uses AP[0] as an access control bit.  */
5372             if ((ap & 1) == 0) {
5373                 /* Access flag fault.  */
5374                 code = (code == 15) ? 6 : 3;
5375                 goto do_fault;
5376             }
5377             *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
5378         } else {
5379             *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
5380         }
5381         if (*prot && !xn) {
5382             *prot |= PAGE_EXEC;
5383         }
5384         if (!(*prot & (1 << access_type))) {
5385             /* Access permission fault.  */
5386             goto do_fault;
5387         }
5388     }
5389     if (ns) {
5390         /* The NS bit will (as required by the architecture) have no effect if
5391          * the CPU doesn't support TZ or this is a non-secure translation
5392          * regime, because the attribute will already be non-secure.
5393          */
5394         attrs->secure = false;
5395     }
5396     *phys_ptr = phys_addr;
5397     return 0;
5398 do_fault:
5399     return code | (domain << 4);
5400 }
5401
5402 /* Fault type for long-descriptor MMU fault reporting; this corresponds
5403  * to bits [5..2] in the STATUS field in long-format DFSR/IFSR.
5404  */
5405 typedef enum {
5406     translation_fault = 1,
5407     access_fault = 2,
5408     permission_fault = 3,
5409 } MMUFaultType;
5410
5411 static int get_phys_addr_lpae(CPUARMState *env, target_ulong address,
5412                               int access_type, ARMMMUIdx mmu_idx,
5413                               hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
5414                               target_ulong *page_size_ptr)
5415 {
5416     CPUState *cs = CPU(arm_env_get_cpu(env));
5417     /* Read an LPAE long-descriptor translation table. */
5418     MMUFaultType fault_type = translation_fault;
5419     uint32_t level = 1;
5420     uint32_t epd;
5421     int32_t tsz;
5422     uint32_t tg;
5423     uint64_t ttbr;
5424     int ttbr_select;
5425     hwaddr descaddr, descmask;
5426     uint32_t tableattrs;
5427     target_ulong page_size;
5428     uint32_t attrs;
5429     int32_t granule_sz = 9;
5430     int32_t va_size = 32;
5431     int32_t tbi = 0;
5432     TCR *tcr = regime_tcr(env, mmu_idx);
5433     int ap, ns, xn, pxn;
5434
5435     /* TODO:
5436      * This code assumes we're either a 64-bit EL1 or a 32-bit PL1;
5437      * it doesn't handle the different format TCR for TCR_EL2, TCR_EL3,
5438      * and VTCR_EL2, or the fact that those regimes don't have a split
5439      * TTBR0/TTBR1. Attribute and permission bit handling should also
5440      * be checked when adding support for those page table walks.
5441      */
5442     if (arm_el_is_aa64(env, regime_el(env, mmu_idx))) {
5443         va_size = 64;
5444         if (extract64(address, 55, 1))
5445             tbi = extract64(tcr->raw_tcr, 38, 1);
5446         else
5447             tbi = extract64(tcr->raw_tcr, 37, 1);
5448         tbi *= 8;
5449     }
5450
5451     /* Determine whether this address is in the region controlled by
5452      * TTBR0 or TTBR1 (or if it is in neither region and should fault).
5453      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
5454      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
5455      */
5456     uint32_t t0sz = extract32(tcr->raw_tcr, 0, 6);
5457     if (va_size == 64) {
5458         t0sz = MIN(t0sz, 39);
5459         t0sz = MAX(t0sz, 16);
5460     }
5461     uint32_t t1sz = extract32(tcr->raw_tcr, 16, 6);
5462     if (va_size == 64) {
5463         t1sz = MIN(t1sz, 39);
5464         t1sz = MAX(t1sz, 16);
5465     }
5466     if (t0sz && !extract64(address, va_size - t0sz, t0sz - tbi)) {
5467         /* there is a ttbr0 region and we are in it (high bits all zero) */
5468         ttbr_select = 0;
5469     } else if (t1sz && !extract64(~address, va_size - t1sz, t1sz - tbi)) {
5470         /* there is a ttbr1 region and we are in it (high bits all one) */
5471         ttbr_select = 1;
5472     } else if (!t0sz) {
5473         /* ttbr0 region is "everything not in the ttbr1 region" */
5474         ttbr_select = 0;
5475     } else if (!t1sz) {
5476         /* ttbr1 region is "everything not in the ttbr0 region" */
5477         ttbr_select = 1;
5478     } else {
5479         /* in the gap between the two regions, this is a Translation fault */
5480         fault_type = translation_fault;
5481         goto do_fault;
5482     }
5483
5484     /* Note that QEMU ignores shareability and cacheability attributes,
5485      * so we don't need to do anything with the SH, ORGN, IRGN fields
5486      * in the TTBCR.  Similarly, TTBCR:A1 selects whether we get the
5487      * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
5488      * implement any ASID-like capability so we can ignore it (instead
5489      * we will always flush the TLB any time the ASID is changed).
5490      */
5491     if (ttbr_select == 0) {
5492         ttbr = A32_BANKED_CURRENT_REG_GET(env, ttbr0);
5493         epd = extract32(tcr->raw_tcr, 7, 1);
5494         tsz = t0sz;
5495
5496         tg = extract32(tcr->raw_tcr, 14, 2);
5497         if (tg == 1) { /* 64KB pages */
5498             granule_sz = 13;
5499         }
5500         if (tg == 2) { /* 16KB pages */
5501             granule_sz = 11;
5502         }
5503     } else {
5504         ttbr = A32_BANKED_CURRENT_REG_GET(env, ttbr1);
5505         epd = extract32(tcr->raw_tcr, 23, 1);
5506         tsz = t1sz;
5507
5508         tg = extract32(tcr->raw_tcr, 30, 2);
5509         if (tg == 3)  { /* 64KB pages */
5510             granule_sz = 13;
5511         }
5512         if (tg == 1) { /* 16KB pages */
5513             granule_sz = 11;
5514         }
5515     }
5516
5517     /* Here we should have set up all the parameters for the translation:
5518      * va_size, ttbr, epd, tsz, granule_sz, tbi
5519      */
5520
5521     if (epd) {
5522         /* Translation table walk disabled => Translation fault on TLB miss */
5523         goto do_fault;
5524     }
5525
5526     /* The starting level depends on the virtual address size (which can be
5527      * up to 48 bits) and the translation granule size. It indicates the number
5528      * of strides (granule_sz bits at a time) needed to consume the bits
5529      * of the input address. In the pseudocode this is:
5530      *  level = 4 - RoundUp((inputsize - grainsize) / stride)
5531      * where their 'inputsize' is our 'va_size - tsz', 'grainsize' is
5532      * our 'granule_sz + 3' and 'stride' is our 'granule_sz'.
5533      * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
5534      *     = 4 - (va_size - tsz - granule_sz - 3 + granule_sz - 1) / granule_sz
5535      *     = 4 - (va_size - tsz - 4) / granule_sz;
5536      */
5537     level = 4 - (va_size - tsz - 4) / granule_sz;
5538
5539     /* Clear the vaddr bits which aren't part of the within-region address,
5540      * so that we don't have to special case things when calculating the
5541      * first descriptor address.
5542      */
5543     if (tsz) {
5544         address &= (1ULL << (va_size - tsz)) - 1;
5545     }
5546
5547     descmask = (1ULL << (granule_sz + 3)) - 1;
5548
5549     /* Now we can extract the actual base address from the TTBR */
5550     descaddr = extract64(ttbr, 0, 48);
5551     descaddr &= ~((1ULL << (va_size - tsz - (granule_sz * (4 - level)))) - 1);
5552
5553     /* Secure accesses start with the page table in secure memory and
5554      * can be downgraded to non-secure at any step. Non-secure accesses
5555      * remain non-secure. We implement this by just ORing in the NSTable/NS
5556      * bits at each step.
5557      */
5558     tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4);
5559     for (;;) {
5560         uint64_t descriptor;
5561         bool nstable;
5562
5563         descaddr |= (address >> (granule_sz * (4 - level))) & descmask;
5564         descaddr &= ~7ULL;
5565         nstable = extract32(tableattrs, 4, 1);
5566         descriptor = arm_ldq_ptw(cs, descaddr, !nstable);
5567         if (!(descriptor & 1) ||
5568             (!(descriptor & 2) && (level == 3))) {
5569             /* Invalid, or the Reserved level 3 encoding */
5570             goto do_fault;
5571         }
5572         descaddr = descriptor & 0xfffffff000ULL;
5573
5574         if ((descriptor & 2) && (level < 3)) {
5575             /* Table entry. The top five bits are attributes which  may
5576              * propagate down through lower levels of the table (and
5577              * which are all arranged so that 0 means "no effect", so
5578              * we can gather them up by ORing in the bits at each level).
5579              */
5580             tableattrs |= extract64(descriptor, 59, 5);
5581             level++;
5582             continue;
5583         }
5584         /* Block entry at level 1 or 2, or page entry at level 3.
5585          * These are basically the same thing, although the number
5586          * of bits we pull in from the vaddr varies.
5587          */
5588         page_size = (1ULL << ((granule_sz * (4 - level)) + 3));
5589         descaddr |= (address & (page_size - 1));
5590         /* Extract attributes from the descriptor and merge with table attrs */
5591         attrs = extract64(descriptor, 2, 10)
5592             | (extract64(descriptor, 52, 12) << 10);
5593         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
5594         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
5595         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
5596          * means "force PL1 access only", which means forcing AP[1] to 0.
5597          */
5598         if (extract32(tableattrs, 2, 1)) {
5599             attrs &= ~(1 << 4);
5600         }
5601         attrs |= nstable << 3; /* NS */
5602         break;
5603     }
5604     /* Here descaddr is the final physical address, and attributes
5605      * are all in attrs.
5606      */
5607     fault_type = access_fault;
5608     if ((attrs & (1 << 8)) == 0) {
5609         /* Access flag */
5610         goto do_fault;
5611     }
5612
5613     ap = extract32(attrs, 4, 2);
5614     ns = extract32(attrs, 3, 1);
5615     xn = extract32(attrs, 12, 1);
5616     pxn = extract32(attrs, 11, 1);
5617
5618     *prot = get_S1prot(env, mmu_idx, va_size == 64, ap, ns, xn, pxn);
5619
5620     fault_type = permission_fault;
5621     if (!(*prot & (1 << access_type))) {
5622         goto do_fault;
5623     }
5624
5625     if (ns) {
5626         /* The NS bit will (as required by the architecture) have no effect if
5627          * the CPU doesn't support TZ or this is a non-secure translation
5628          * regime, because the attribute will already be non-secure.
5629          */
5630         txattrs->secure = false;
5631     }
5632     *phys_ptr = descaddr;
5633     *page_size_ptr = page_size;
5634     return 0;
5635
5636 do_fault:
5637     /* Long-descriptor format IFSR/DFSR value */
5638     return (1 << 9) | (fault_type << 2) | level;
5639 }
5640
5641 static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
5642                              int access_type, ARMMMUIdx mmu_idx,
5643                              hwaddr *phys_ptr, int *prot)
5644 {
5645     int n;
5646     uint32_t mask;
5647     uint32_t base;
5648     bool is_user = regime_is_user(env, mmu_idx);
5649
5650     *phys_ptr = address;
5651     for (n = 7; n >= 0; n--) {
5652         base = env->cp15.c6_region[n];
5653         if ((base & 1) == 0) {
5654             continue;
5655         }
5656         mask = 1 << ((base >> 1) & 0x1f);
5657         /* Keep this shift separate from the above to avoid an
5658            (undefined) << 32.  */
5659         mask = (mask << 1) - 1;
5660         if (((base ^ address) & ~mask) == 0) {
5661             break;
5662         }
5663     }
5664     if (n < 0) {
5665         return 2;
5666     }
5667
5668     if (access_type == 2) {
5669         mask = env->cp15.pmsav5_insn_ap;
5670     } else {
5671         mask = env->cp15.pmsav5_data_ap;
5672     }
5673     mask = (mask >> (n * 4)) & 0xf;
5674     switch (mask) {
5675     case 0:
5676         return 1;
5677     case 1:
5678         if (is_user) {
5679             return 1;
5680         }
5681         *prot = PAGE_READ | PAGE_WRITE;
5682         break;
5683     case 2:
5684         *prot = PAGE_READ;
5685         if (!is_user) {
5686             *prot |= PAGE_WRITE;
5687         }
5688         break;
5689     case 3:
5690         *prot = PAGE_READ | PAGE_WRITE;
5691         break;
5692     case 5:
5693         if (is_user) {
5694             return 1;
5695         }
5696         *prot = PAGE_READ;
5697         break;
5698     case 6:
5699         *prot = PAGE_READ;
5700         break;
5701     default:
5702         /* Bad permission.  */
5703         return 1;
5704     }
5705     *prot |= PAGE_EXEC;
5706     return 0;
5707 }
5708
5709 /* get_phys_addr - get the physical address for this virtual address
5710  *
5711  * Find the physical address corresponding to the given virtual address,
5712  * by doing a translation table walk on MMU based systems or using the
5713  * MPU state on MPU based systems.
5714  *
5715  * Returns 0 if the translation was successful. Otherwise, phys_ptr, attrs,
5716  * prot and page_size may not be filled in, and the return value provides
5717  * information on why the translation aborted, in the format of a
5718  * DFSR/IFSR fault register, with the following caveats:
5719  *  * we honour the short vs long DFSR format differences.
5720  *  * the WnR bit is never set (the caller must do this).
5721  *  * for MPU based systems we don't bother to return a full FSR format
5722  *    value.
5723  *
5724  * @env: CPUARMState
5725  * @address: virtual address to get physical address for
5726  * @access_type: 0 for read, 1 for write, 2 for execute
5727  * @mmu_idx: MMU index indicating required translation regime
5728  * @phys_ptr: set to the physical address corresponding to the virtual address
5729  * @attrs: set to the memory transaction attributes to use
5730  * @prot: set to the permissions for the page containing phys_ptr
5731  * @page_size: set to the size of the page containing phys_ptr
5732  */
5733 static inline int get_phys_addr(CPUARMState *env, target_ulong address,
5734                                 int access_type, ARMMMUIdx mmu_idx,
5735                                 hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
5736                                 target_ulong *page_size)
5737 {
5738     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
5739         /* TODO: when we support EL2 we should here call ourselves recursively
5740          * to do the stage 1 and then stage 2 translations. The arm_ld*_ptw
5741          * functions will also need changing to perform ARMMMUIdx_S2NS loads
5742          * rather than direct physical memory loads when appropriate.
5743          * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
5744          */
5745         assert(!arm_feature(env, ARM_FEATURE_EL2));
5746         mmu_idx += ARMMMUIdx_S1NSE0;
5747     }
5748
5749     /* The page table entries may downgrade secure to non-secure, but
5750      * cannot upgrade an non-secure translation regime's attributes
5751      * to secure.
5752      */
5753     attrs->secure = regime_is_secure(env, mmu_idx);
5754     attrs->user = regime_is_user(env, mmu_idx);
5755
5756     /* Fast Context Switch Extension. This doesn't exist at all in v8.
5757      * In v7 and earlier it affects all stage 1 translations.
5758      */
5759     if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS
5760         && !arm_feature(env, ARM_FEATURE_V8)) {
5761         if (regime_el(env, mmu_idx) == 3) {
5762             address += env->cp15.fcseidr_s;
5763         } else {
5764             address += env->cp15.fcseidr_ns;
5765         }
5766     }
5767
5768     if (regime_translation_disabled(env, mmu_idx)) {
5769         /* MMU/MPU disabled.  */
5770         *phys_ptr = address;
5771         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
5772         *page_size = TARGET_PAGE_SIZE;
5773         return 0;
5774     }
5775
5776     if (arm_feature(env, ARM_FEATURE_MPU)) {
5777         *page_size = TARGET_PAGE_SIZE;
5778         return get_phys_addr_mpu(env, address, access_type, mmu_idx, phys_ptr,
5779                                  prot);
5780     }
5781
5782     if (regime_using_lpae_format(env, mmu_idx)) {
5783         return get_phys_addr_lpae(env, address, access_type, mmu_idx, phys_ptr,
5784                                   attrs, prot, page_size);
5785     } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
5786         return get_phys_addr_v6(env, address, access_type, mmu_idx, phys_ptr,
5787                                 attrs, prot, page_size);
5788     } else {
5789         return get_phys_addr_v5(env, address, access_type, mmu_idx, phys_ptr,
5790                                 prot, page_size);
5791     }
5792 }
5793
5794 int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
5795                              int access_type, int mmu_idx)
5796 {
5797     ARMCPU *cpu = ARM_CPU(cs);
5798     CPUARMState *env = &cpu->env;
5799     hwaddr phys_addr;
5800     target_ulong page_size;
5801     int prot;
5802     int ret;
5803     uint32_t syn;
5804     bool same_el = (arm_current_el(env) != 0);
5805     MemTxAttrs attrs = {};
5806
5807     ret = get_phys_addr(env, address, access_type, mmu_idx, &phys_addr,
5808                         &attrs, &prot, &page_size);
5809     if (ret == 0) {
5810         /* Map a single [sub]page.  */
5811         phys_addr &= TARGET_PAGE_MASK;
5812         address &= TARGET_PAGE_MASK;
5813         tlb_set_page_with_attrs(cs, address, phys_addr, attrs,
5814                                 prot, mmu_idx, page_size);
5815         return 0;
5816     }
5817
5818     /* AArch64 syndrome does not have an LPAE bit */
5819     syn = ret & ~(1 << 9);
5820
5821     /* For insn and data aborts we assume there is no instruction syndrome
5822      * information; this is always true for exceptions reported to EL1.
5823      */
5824     if (access_type == 2) {
5825         syn = syn_insn_abort(same_el, 0, 0, syn);
5826         cs->exception_index = EXCP_PREFETCH_ABORT;
5827     } else {
5828         syn = syn_data_abort(same_el, 0, 0, 0, access_type == 1, syn);
5829         if (access_type == 1 && arm_feature(env, ARM_FEATURE_V6)) {
5830             ret |= (1 << 11);
5831         }
5832         cs->exception_index = EXCP_DATA_ABORT;
5833     }
5834
5835     env->exception.syndrome = syn;
5836     env->exception.vaddress = address;
5837     env->exception.fsr = ret;
5838     return 1;
5839 }
5840
5841 hwaddr arm_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
5842 {
5843     ARMCPU *cpu = ARM_CPU(cs);
5844     CPUARMState *env = &cpu->env;
5845     hwaddr phys_addr;
5846     target_ulong page_size;
5847     int prot;
5848     int ret;
5849     MemTxAttrs attrs = {};
5850
5851     ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env), &phys_addr,
5852                         &attrs, &prot, &page_size);
5853
5854     if (ret != 0) {
5855         return -1;
5856     }
5857
5858     return phys_addr;
5859 }
5860
5861 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
5862 {
5863     if ((env->uncached_cpsr & CPSR_M) == mode) {
5864         env->regs[13] = val;
5865     } else {
5866         env->banked_r13[bank_number(mode)] = val;
5867     }
5868 }
5869
5870 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
5871 {
5872     if ((env->uncached_cpsr & CPSR_M) == mode) {
5873         return env->regs[13];
5874     } else {
5875         return env->banked_r13[bank_number(mode)];
5876     }
5877 }
5878
5879 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
5880 {
5881     ARMCPU *cpu = arm_env_get_cpu(env);
5882
5883     switch (reg) {
5884     case 0: /* APSR */
5885         return xpsr_read(env) & 0xf8000000;
5886     case 1: /* IAPSR */
5887         return xpsr_read(env) & 0xf80001ff;
5888     case 2: /* EAPSR */
5889         return xpsr_read(env) & 0xff00fc00;
5890     case 3: /* xPSR */
5891         return xpsr_read(env) & 0xff00fdff;
5892     case 5: /* IPSR */
5893         return xpsr_read(env) & 0x000001ff;
5894     case 6: /* EPSR */
5895         return xpsr_read(env) & 0x0700fc00;
5896     case 7: /* IEPSR */
5897         return xpsr_read(env) & 0x0700edff;
5898     case 8: /* MSP */
5899         return env->v7m.current_sp ? env->v7m.other_sp : env->regs[13];
5900     case 9: /* PSP */
5901         return env->v7m.current_sp ? env->regs[13] : env->v7m.other_sp;
5902     case 16: /* PRIMASK */
5903         return (env->daif & PSTATE_I) != 0;
5904     case 17: /* BASEPRI */
5905     case 18: /* BASEPRI_MAX */
5906         return env->v7m.basepri;
5907     case 19: /* FAULTMASK */
5908         return (env->daif & PSTATE_F) != 0;
5909     case 20: /* CONTROL */
5910         return env->v7m.control;
5911     default:
5912         /* ??? For debugging only.  */
5913         cpu_abort(CPU(cpu), "Unimplemented system register read (%d)\n", reg);
5914         return 0;
5915     }
5916 }
5917
5918 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
5919 {
5920     ARMCPU *cpu = arm_env_get_cpu(env);
5921
5922     switch (reg) {
5923     case 0: /* APSR */
5924         xpsr_write(env, val, 0xf8000000);
5925         break;
5926     case 1: /* IAPSR */
5927         xpsr_write(env, val, 0xf8000000);
5928         break;
5929     case 2: /* EAPSR */
5930         xpsr_write(env, val, 0xfe00fc00);
5931         break;
5932     case 3: /* xPSR */
5933         xpsr_write(env, val, 0xfe00fc00);
5934         break;
5935     case 5: /* IPSR */
5936         /* IPSR bits are readonly.  */
5937         break;
5938     case 6: /* EPSR */
5939         xpsr_write(env, val, 0x0600fc00);
5940         break;
5941     case 7: /* IEPSR */
5942         xpsr_write(env, val, 0x0600fc00);
5943         break;
5944     case 8: /* MSP */
5945         if (env->v7m.current_sp)
5946             env->v7m.other_sp = val;
5947         else
5948             env->regs[13] = val;
5949         break;
5950     case 9: /* PSP */
5951         if (env->v7m.current_sp)
5952             env->regs[13] = val;
5953         else
5954             env->v7m.other_sp = val;
5955         break;
5956     case 16: /* PRIMASK */
5957         if (val & 1) {
5958             env->daif |= PSTATE_I;
5959         } else {
5960             env->daif &= ~PSTATE_I;
5961         }
5962         break;
5963     case 17: /* BASEPRI */
5964         env->v7m.basepri = val & 0xff;
5965         break;
5966     case 18: /* BASEPRI_MAX */
5967         val &= 0xff;
5968         if (val != 0 && (val < env->v7m.basepri || env->v7m.basepri == 0))
5969             env->v7m.basepri = val;
5970         break;
5971     case 19: /* FAULTMASK */
5972         if (val & 1) {
5973             env->daif |= PSTATE_F;
5974         } else {
5975             env->daif &= ~PSTATE_F;
5976         }
5977         break;
5978     case 20: /* CONTROL */
5979         env->v7m.control = val & 3;
5980         switch_v7m_sp(env, (val & 2) != 0);
5981         break;
5982     default:
5983         /* ??? For debugging only.  */
5984         cpu_abort(CPU(cpu), "Unimplemented system register write (%d)\n", reg);
5985         return;
5986     }
5987 }
5988
5989 #endif
5990
5991 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
5992 {
5993     /* Implement DC ZVA, which zeroes a fixed-length block of memory.
5994      * Note that we do not implement the (architecturally mandated)
5995      * alignment fault for attempts to use this on Device memory
5996      * (which matches the usual QEMU behaviour of not implementing either
5997      * alignment faults or any memory attribute handling).
5998      */
5999
6000     ARMCPU *cpu = arm_env_get_cpu(env);
6001     uint64_t blocklen = 4 << cpu->dcz_blocksize;
6002     uint64_t vaddr = vaddr_in & ~(blocklen - 1);
6003
6004 #ifndef CONFIG_USER_ONLY
6005     {
6006         /* Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than
6007          * the block size so we might have to do more than one TLB lookup.
6008          * We know that in fact for any v8 CPU the page size is at least 4K
6009          * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only
6010          * 1K as an artefact of legacy v5 subpage support being present in the
6011          * same QEMU executable.
6012          */
6013         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
6014         void *hostaddr[maxidx];
6015         int try, i;
6016
6017         for (try = 0; try < 2; try++) {
6018
6019             for (i = 0; i < maxidx; i++) {
6020                 hostaddr[i] = tlb_vaddr_to_host(env,
6021                                                 vaddr + TARGET_PAGE_SIZE * i,
6022                                                 1, cpu_mmu_index(env));
6023                 if (!hostaddr[i]) {
6024                     break;
6025                 }
6026             }
6027             if (i == maxidx) {
6028                 /* If it's all in the TLB it's fair game for just writing to;
6029                  * we know we don't need to update dirty status, etc.
6030                  */
6031                 for (i = 0; i < maxidx - 1; i++) {
6032                     memset(hostaddr[i], 0, TARGET_PAGE_SIZE);
6033                 }
6034                 memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE));
6035                 return;
6036             }
6037             /* OK, try a store and see if we can populate the tlb. This
6038              * might cause an exception if the memory isn't writable,
6039              * in which case we will longjmp out of here. We must for
6040              * this purpose use the actual register value passed to us
6041              * so that we get the fault address right.
6042              */
6043             helper_ret_stb_mmu(env, vaddr_in, 0, cpu_mmu_index(env), GETRA());
6044             /* Now we can populate the other TLB entries, if any */
6045             for (i = 0; i < maxidx; i++) {
6046                 uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
6047                 if (va != (vaddr_in & TARGET_PAGE_MASK)) {
6048                     helper_ret_stb_mmu(env, va, 0, cpu_mmu_index(env), GETRA());
6049                 }
6050             }
6051         }
6052
6053         /* Slow path (probably attempt to do this to an I/O device or
6054          * similar, or clearing of a block of code we have translations
6055          * cached for). Just do a series of byte writes as the architecture
6056          * demands. It's not worth trying to use a cpu_physical_memory_map(),
6057          * memset(), unmap() sequence here because:
6058          *  + we'd need to account for the blocksize being larger than a page
6059          *  + the direct-RAM access case is almost always going to be dealt
6060          *    with in the fastpath code above, so there's no speed benefit
6061          *  + we would have to deal with the map returning NULL because the
6062          *    bounce buffer was in use
6063          */
6064         for (i = 0; i < blocklen; i++) {
6065             helper_ret_stb_mmu(env, vaddr + i, 0, cpu_mmu_index(env), GETRA());
6066         }
6067     }
6068 #else
6069     memset(g2h(vaddr), 0, blocklen);
6070 #endif
6071 }
6072
6073 /* Note that signed overflow is undefined in C.  The following routines are
6074    careful to use unsigned types where modulo arithmetic is required.
6075    Failure to do so _will_ break on newer gcc.  */
6076
6077 /* Signed saturating arithmetic.  */
6078
6079 /* Perform 16-bit signed saturating addition.  */
6080 static inline uint16_t add16_sat(uint16_t a, uint16_t b)
6081 {
6082     uint16_t res;
6083
6084     res = a + b;
6085     if (((res ^ a) & 0x8000) && !((a ^ b) & 0x8000)) {
6086         if (a & 0x8000)
6087             res = 0x8000;
6088         else
6089             res = 0x7fff;
6090     }
6091     return res;
6092 }
6093
6094 /* Perform 8-bit signed saturating addition.  */
6095 static inline uint8_t add8_sat(uint8_t a, uint8_t b)
6096 {
6097     uint8_t res;
6098
6099     res = a + b;
6100     if (((res ^ a) & 0x80) && !((a ^ b) & 0x80)) {
6101         if (a & 0x80)
6102             res = 0x80;
6103         else
6104             res = 0x7f;
6105     }
6106     return res;
6107 }
6108
6109 /* Perform 16-bit signed saturating subtraction.  */
6110 static inline uint16_t sub16_sat(uint16_t a, uint16_t b)
6111 {
6112     uint16_t res;
6113
6114     res = a - b;
6115     if (((res ^ a) & 0x8000) && ((a ^ b) & 0x8000)) {
6116         if (a & 0x8000)
6117             res = 0x8000;
6118         else
6119             res = 0x7fff;
6120     }
6121     return res;
6122 }
6123
6124 /* Perform 8-bit signed saturating subtraction.  */
6125 static inline uint8_t sub8_sat(uint8_t a, uint8_t b)
6126 {
6127     uint8_t res;
6128
6129     res = a - b;
6130     if (((res ^ a) & 0x80) && ((a ^ b) & 0x80)) {
6131         if (a & 0x80)
6132             res = 0x80;
6133         else
6134             res = 0x7f;
6135     }
6136     return res;
6137 }
6138
6139 #define ADD16(a, b, n) RESULT(add16_sat(a, b), n, 16);
6140 #define SUB16(a, b, n) RESULT(sub16_sat(a, b), n, 16);
6141 #define ADD8(a, b, n)  RESULT(add8_sat(a, b), n, 8);
6142 #define SUB8(a, b, n)  RESULT(sub8_sat(a, b), n, 8);
6143 #define PFX q
6144
6145 #include "op_addsub.h"
6146
6147 /* Unsigned saturating arithmetic.  */
6148 static inline uint16_t add16_usat(uint16_t a, uint16_t b)
6149 {
6150     uint16_t res;
6151     res = a + b;
6152     if (res < a)
6153         res = 0xffff;
6154     return res;
6155 }
6156
6157 static inline uint16_t sub16_usat(uint16_t a, uint16_t b)
6158 {
6159     if (a > b)
6160         return a - b;
6161     else
6162         return 0;
6163 }
6164
6165 static inline uint8_t add8_usat(uint8_t a, uint8_t b)
6166 {
6167     uint8_t res;
6168     res = a + b;
6169     if (res < a)
6170         res = 0xff;
6171     return res;
6172 }
6173
6174 static inline uint8_t sub8_usat(uint8_t a, uint8_t b)
6175 {
6176     if (a > b)
6177         return a - b;
6178     else
6179         return 0;
6180 }
6181
6182 #define ADD16(a, b, n) RESULT(add16_usat(a, b), n, 16);
6183 #define SUB16(a, b, n) RESULT(sub16_usat(a, b), n, 16);
6184 #define ADD8(a, b, n)  RESULT(add8_usat(a, b), n, 8);
6185 #define SUB8(a, b, n)  RESULT(sub8_usat(a, b), n, 8);
6186 #define PFX uq
6187
6188 #include "op_addsub.h"
6189
6190 /* Signed modulo arithmetic.  */
6191 #define SARITH16(a, b, n, op) do { \
6192     int32_t sum; \
6193     sum = (int32_t)(int16_t)(a) op (int32_t)(int16_t)(b); \
6194     RESULT(sum, n, 16); \
6195     if (sum >= 0) \
6196         ge |= 3 << (n * 2); \
6197     } while(0)
6198
6199 #define SARITH8(a, b, n, op) do { \
6200     int32_t sum; \
6201     sum = (int32_t)(int8_t)(a) op (int32_t)(int8_t)(b); \
6202     RESULT(sum, n, 8); \
6203     if (sum >= 0) \
6204         ge |= 1 << n; \
6205     } while(0)
6206
6207
6208 #define ADD16(a, b, n) SARITH16(a, b, n, +)
6209 #define SUB16(a, b, n) SARITH16(a, b, n, -)
6210 #define ADD8(a, b, n)  SARITH8(a, b, n, +)
6211 #define SUB8(a, b, n)  SARITH8(a, b, n, -)
6212 #define PFX s
6213 #define ARITH_GE
6214
6215 #include "op_addsub.h"
6216
6217 /* Unsigned modulo arithmetic.  */
6218 #define ADD16(a, b, n) do { \
6219     uint32_t sum; \
6220     sum = (uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b); \
6221     RESULT(sum, n, 16); \
6222     if ((sum >> 16) == 1) \
6223         ge |= 3 << (n * 2); \
6224     } while(0)
6225
6226 #define ADD8(a, b, n) do { \
6227     uint32_t sum; \
6228     sum = (uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b); \
6229     RESULT(sum, n, 8); \
6230     if ((sum >> 8) == 1) \
6231         ge |= 1 << n; \
6232     } while(0)
6233
6234 #define SUB16(a, b, n) do { \
6235     uint32_t sum; \
6236     sum = (uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b); \
6237     RESULT(sum, n, 16); \
6238     if ((sum >> 16) == 0) \
6239         ge |= 3 << (n * 2); \
6240     } while(0)
6241
6242 #define SUB8(a, b, n) do { \
6243     uint32_t sum; \
6244     sum = (uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b); \
6245     RESULT(sum, n, 8); \
6246     if ((sum >> 8) == 0) \
6247         ge |= 1 << n; \
6248     } while(0)
6249
6250 #define PFX u
6251 #define ARITH_GE
6252
6253 #include "op_addsub.h"
6254
6255 /* Halved signed arithmetic.  */
6256 #define ADD16(a, b, n) \
6257   RESULT(((int32_t)(int16_t)(a) + (int32_t)(int16_t)(b)) >> 1, n, 16)
6258 #define SUB16(a, b, n) \
6259   RESULT(((int32_t)(int16_t)(a) - (int32_t)(int16_t)(b)) >> 1, n, 16)
6260 #define ADD8(a, b, n) \
6261   RESULT(((int32_t)(int8_t)(a) + (int32_t)(int8_t)(b)) >> 1, n, 8)
6262 #define SUB8(a, b, n) \
6263   RESULT(((int32_t)(int8_t)(a) - (int32_t)(int8_t)(b)) >> 1, n, 8)
6264 #define PFX sh
6265
6266 #include "op_addsub.h"
6267
6268 /* Halved unsigned arithmetic.  */
6269 #define ADD16(a, b, n) \
6270   RESULT(((uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b)) >> 1, n, 16)
6271 #define SUB16(a, b, n) \
6272   RESULT(((uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b)) >> 1, n, 16)
6273 #define ADD8(a, b, n) \
6274   RESULT(((uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b)) >> 1, n, 8)
6275 #define SUB8(a, b, n) \
6276   RESULT(((uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b)) >> 1, n, 8)
6277 #define PFX uh
6278
6279 #include "op_addsub.h"
6280
6281 static inline uint8_t do_usad(uint8_t a, uint8_t b)
6282 {
6283     if (a > b)
6284         return a - b;
6285     else
6286         return b - a;
6287 }
6288
6289 /* Unsigned sum of absolute byte differences.  */
6290 uint32_t HELPER(usad8)(uint32_t a, uint32_t b)
6291 {
6292     uint32_t sum;
6293     sum = do_usad(a, b);
6294     sum += do_usad(a >> 8, b >> 8);
6295     sum += do_usad(a >> 16, b >>16);
6296     sum += do_usad(a >> 24, b >> 24);
6297     return sum;
6298 }
6299
6300 /* For ARMv6 SEL instruction.  */
6301 uint32_t HELPER(sel_flags)(uint32_t flags, uint32_t a, uint32_t b)
6302 {
6303     uint32_t mask;
6304
6305     mask = 0;
6306     if (flags & 1)
6307         mask |= 0xff;
6308     if (flags & 2)
6309         mask |= 0xff00;
6310     if (flags & 4)
6311         mask |= 0xff0000;
6312     if (flags & 8)
6313         mask |= 0xff000000;
6314     return (a & mask) | (b & ~mask);
6315 }
6316
6317 /* VFP support.  We follow the convention used for VFP instructions:
6318    Single precision routines have a "s" suffix, double precision a
6319    "d" suffix.  */
6320
6321 /* Convert host exception flags to vfp form.  */
6322 static inline int vfp_exceptbits_from_host(int host_bits)
6323 {
6324     int target_bits = 0;
6325
6326     if (host_bits & float_flag_invalid)
6327         target_bits |= 1;
6328     if (host_bits & float_flag_divbyzero)
6329         target_bits |= 2;
6330     if (host_bits & float_flag_overflow)
6331         target_bits |= 4;
6332     if (host_bits & (float_flag_underflow | float_flag_output_denormal))
6333         target_bits |= 8;
6334     if (host_bits & float_flag_inexact)
6335         target_bits |= 0x10;
6336     if (host_bits & float_flag_input_denormal)
6337         target_bits |= 0x80;
6338     return target_bits;
6339 }
6340
6341 uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env)
6342 {
6343     int i;
6344     uint32_t fpscr;
6345
6346     fpscr = (env->vfp.xregs[ARM_VFP_FPSCR] & 0xffc8ffff)
6347             | (env->vfp.vec_len << 16)
6348             | (env->vfp.vec_stride << 20);
6349     i = get_float_exception_flags(&env->vfp.fp_status);
6350     i |= get_float_exception_flags(&env->vfp.standard_fp_status);
6351     fpscr |= vfp_exceptbits_from_host(i);
6352     return fpscr;
6353 }
6354
6355 uint32_t vfp_get_fpscr(CPUARMState *env)
6356 {
6357     return HELPER(vfp_get_fpscr)(env);
6358 }
6359
6360 /* Convert vfp exception flags to target form.  */
6361 static inline int vfp_exceptbits_to_host(int target_bits)
6362 {
6363     int host_bits = 0;
6364
6365     if (target_bits & 1)
6366         host_bits |= float_flag_invalid;
6367     if (target_bits & 2)
6368         host_bits |= float_flag_divbyzero;
6369     if (target_bits & 4)
6370         host_bits |= float_flag_overflow;
6371     if (target_bits & 8)
6372         host_bits |= float_flag_underflow;
6373     if (target_bits & 0x10)
6374         host_bits |= float_flag_inexact;
6375     if (target_bits & 0x80)
6376         host_bits |= float_flag_input_denormal;
6377     return host_bits;
6378 }
6379
6380 void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val)
6381 {
6382     int i;
6383     uint32_t changed;
6384
6385     changed = env->vfp.xregs[ARM_VFP_FPSCR];
6386     env->vfp.xregs[ARM_VFP_FPSCR] = (val & 0xffc8ffff);
6387     env->vfp.vec_len = (val >> 16) & 7;
6388     env->vfp.vec_stride = (val >> 20) & 3;
6389
6390     changed ^= val;
6391     if (changed & (3 << 22)) {
6392         i = (val >> 22) & 3;
6393         switch (i) {
6394         case FPROUNDING_TIEEVEN:
6395             i = float_round_nearest_even;
6396             break;
6397         case FPROUNDING_POSINF:
6398             i = float_round_up;
6399             break;
6400         case FPROUNDING_NEGINF:
6401             i = float_round_down;
6402             break;
6403         case FPROUNDING_ZERO:
6404             i = float_round_to_zero;
6405             break;
6406         }
6407         set_float_rounding_mode(i, &env->vfp.fp_status);
6408     }
6409     if (changed & (1 << 24)) {
6410         set_flush_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
6411         set_flush_inputs_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
6412     }
6413     if (changed & (1 << 25))
6414         set_default_nan_mode((val & (1 << 25)) != 0, &env->vfp.fp_status);
6415
6416     i = vfp_exceptbits_to_host(val);
6417     set_float_exception_flags(i, &env->vfp.fp_status);
6418     set_float_exception_flags(0, &env->vfp.standard_fp_status);
6419 }
6420
6421 void vfp_set_fpscr(CPUARMState *env, uint32_t val)
6422 {
6423     HELPER(vfp_set_fpscr)(env, val);
6424 }
6425
6426 #define VFP_HELPER(name, p) HELPER(glue(glue(vfp_,name),p))
6427
6428 #define VFP_BINOP(name) \
6429 float32 VFP_HELPER(name, s)(float32 a, float32 b, void *fpstp) \
6430 { \
6431     float_status *fpst = fpstp; \
6432     return float32_ ## name(a, b, fpst); \
6433 } \
6434 float64 VFP_HELPER(name, d)(float64 a, float64 b, void *fpstp) \
6435 { \
6436     float_status *fpst = fpstp; \
6437     return float64_ ## name(a, b, fpst); \
6438 }
6439 VFP_BINOP(add)
6440 VFP_BINOP(sub)
6441 VFP_BINOP(mul)
6442 VFP_BINOP(div)
6443 VFP_BINOP(min)
6444 VFP_BINOP(max)
6445 VFP_BINOP(minnum)
6446 VFP_BINOP(maxnum)
6447 #undef VFP_BINOP
6448
6449 float32 VFP_HELPER(neg, s)(float32 a)
6450 {
6451     return float32_chs(a);
6452 }
6453
6454 float64 VFP_HELPER(neg, d)(float64 a)
6455 {
6456     return float64_chs(a);
6457 }
6458
6459 float32 VFP_HELPER(abs, s)(float32 a)
6460 {
6461     return float32_abs(a);
6462 }
6463
6464 float64 VFP_HELPER(abs, d)(float64 a)
6465 {
6466     return float64_abs(a);
6467 }
6468
6469 float32 VFP_HELPER(sqrt, s)(float32 a, CPUARMState *env)
6470 {
6471     return float32_sqrt(a, &env->vfp.fp_status);
6472 }
6473
6474 float64 VFP_HELPER(sqrt, d)(float64 a, CPUARMState *env)
6475 {
6476     return float64_sqrt(a, &env->vfp.fp_status);
6477 }
6478
6479 /* XXX: check quiet/signaling case */
6480 #define DO_VFP_cmp(p, type) \
6481 void VFP_HELPER(cmp, p)(type a, type b, CPUARMState *env)  \
6482 { \
6483     uint32_t flags; \
6484     switch(type ## _compare_quiet(a, b, &env->vfp.fp_status)) { \
6485     case 0: flags = 0x6; break; \
6486     case -1: flags = 0x8; break; \
6487     case 1: flags = 0x2; break; \
6488     default: case 2: flags = 0x3; break; \
6489     } \
6490     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
6491         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
6492 } \
6493 void VFP_HELPER(cmpe, p)(type a, type b, CPUARMState *env) \
6494 { \
6495     uint32_t flags; \
6496     switch(type ## _compare(a, b, &env->vfp.fp_status)) { \
6497     case 0: flags = 0x6; break; \
6498     case -1: flags = 0x8; break; \
6499     case 1: flags = 0x2; break; \
6500     default: case 2: flags = 0x3; break; \
6501     } \
6502     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
6503         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
6504 }
6505 DO_VFP_cmp(s, float32)
6506 DO_VFP_cmp(d, float64)
6507 #undef DO_VFP_cmp
6508
6509 /* Integer to float and float to integer conversions */
6510
6511 #define CONV_ITOF(name, fsz, sign) \
6512     float##fsz HELPER(name)(uint32_t x, void *fpstp) \
6513 { \
6514     float_status *fpst = fpstp; \
6515     return sign##int32_to_##float##fsz((sign##int32_t)x, fpst); \
6516 }
6517
6518 #define CONV_FTOI(name, fsz, sign, round) \
6519 uint32_t HELPER(name)(float##fsz x, void *fpstp) \
6520 { \
6521     float_status *fpst = fpstp; \
6522     if (float##fsz##_is_any_nan(x)) { \
6523         float_raise(float_flag_invalid, fpst); \
6524         return 0; \
6525     } \
6526     return float##fsz##_to_##sign##int32##round(x, fpst); \
6527 }
6528
6529 #define FLOAT_CONVS(name, p, fsz, sign) \
6530 CONV_ITOF(vfp_##name##to##p, fsz, sign) \
6531 CONV_FTOI(vfp_to##name##p, fsz, sign, ) \
6532 CONV_FTOI(vfp_to##name##z##p, fsz, sign, _round_to_zero)
6533
6534 FLOAT_CONVS(si, s, 32, )
6535 FLOAT_CONVS(si, d, 64, )
6536 FLOAT_CONVS(ui, s, 32, u)
6537 FLOAT_CONVS(ui, d, 64, u)
6538
6539 #undef CONV_ITOF
6540 #undef CONV_FTOI
6541 #undef FLOAT_CONVS
6542
6543 /* floating point conversion */
6544 float64 VFP_HELPER(fcvtd, s)(float32 x, CPUARMState *env)
6545 {
6546     float64 r = float32_to_float64(x, &env->vfp.fp_status);
6547     /* ARM requires that S<->D conversion of any kind of NaN generates
6548      * a quiet NaN by forcing the most significant frac bit to 1.
6549      */
6550     return float64_maybe_silence_nan(r);
6551 }
6552
6553 float32 VFP_HELPER(fcvts, d)(float64 x, CPUARMState *env)
6554 {
6555     float32 r =  float64_to_float32(x, &env->vfp.fp_status);
6556     /* ARM requires that S<->D conversion of any kind of NaN generates
6557      * a quiet NaN by forcing the most significant frac bit to 1.
6558      */
6559     return float32_maybe_silence_nan(r);
6560 }
6561
6562 /* VFP3 fixed point conversion.  */
6563 #define VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype) \
6564 float##fsz HELPER(vfp_##name##to##p)(uint##isz##_t  x, uint32_t shift, \
6565                                      void *fpstp) \
6566 { \
6567     float_status *fpst = fpstp; \
6568     float##fsz tmp; \
6569     tmp = itype##_to_##float##fsz(x, fpst); \
6570     return float##fsz##_scalbn(tmp, -(int)shift, fpst); \
6571 }
6572
6573 /* Notice that we want only input-denormal exception flags from the
6574  * scalbn operation: the other possible flags (overflow+inexact if
6575  * we overflow to infinity, output-denormal) aren't correct for the
6576  * complete scale-and-convert operation.
6577  */
6578 #define VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, round) \
6579 uint##isz##_t HELPER(vfp_to##name##p##round)(float##fsz x, \
6580                                              uint32_t shift, \
6581                                              void *fpstp) \
6582 { \
6583     float_status *fpst = fpstp; \
6584     int old_exc_flags = get_float_exception_flags(fpst); \
6585     float##fsz tmp; \
6586     if (float##fsz##_is_any_nan(x)) { \
6587         float_raise(float_flag_invalid, fpst); \
6588         return 0; \
6589     } \
6590     tmp = float##fsz##_scalbn(x, shift, fpst); \
6591     old_exc_flags |= get_float_exception_flags(fpst) \
6592         & float_flag_input_denormal; \
6593     set_float_exception_flags(old_exc_flags, fpst); \
6594     return float##fsz##_to_##itype##round(tmp, fpst); \
6595 }
6596
6597 #define VFP_CONV_FIX(name, p, fsz, isz, itype)                   \
6598 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
6599 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, _round_to_zero) \
6600 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
6601
6602 #define VFP_CONV_FIX_A64(name, p, fsz, isz, itype)               \
6603 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
6604 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
6605
6606 VFP_CONV_FIX(sh, d, 64, 64, int16)
6607 VFP_CONV_FIX(sl, d, 64, 64, int32)
6608 VFP_CONV_FIX_A64(sq, d, 64, 64, int64)
6609 VFP_CONV_FIX(uh, d, 64, 64, uint16)
6610 VFP_CONV_FIX(ul, d, 64, 64, uint32)
6611 VFP_CONV_FIX_A64(uq, d, 64, 64, uint64)
6612 VFP_CONV_FIX(sh, s, 32, 32, int16)
6613 VFP_CONV_FIX(sl, s, 32, 32, int32)
6614 VFP_CONV_FIX_A64(sq, s, 32, 64, int64)
6615 VFP_CONV_FIX(uh, s, 32, 32, uint16)
6616 VFP_CONV_FIX(ul, s, 32, 32, uint32)
6617 VFP_CONV_FIX_A64(uq, s, 32, 64, uint64)
6618 #undef VFP_CONV_FIX
6619 #undef VFP_CONV_FIX_FLOAT
6620 #undef VFP_CONV_FLOAT_FIX_ROUND
6621
6622 /* Set the current fp rounding mode and return the old one.
6623  * The argument is a softfloat float_round_ value.
6624  */
6625 uint32_t HELPER(set_rmode)(uint32_t rmode, CPUARMState *env)
6626 {
6627     float_status *fp_status = &env->vfp.fp_status;
6628
6629     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
6630     set_float_rounding_mode(rmode, fp_status);
6631
6632     return prev_rmode;
6633 }
6634
6635 /* Set the current fp rounding mode in the standard fp status and return
6636  * the old one. This is for NEON instructions that need to change the
6637  * rounding mode but wish to use the standard FPSCR values for everything
6638  * else. Always set the rounding mode back to the correct value after
6639  * modifying it.
6640  * The argument is a softfloat float_round_ value.
6641  */
6642 uint32_t HELPER(set_neon_rmode)(uint32_t rmode, CPUARMState *env)
6643 {
6644     float_status *fp_status = &env->vfp.standard_fp_status;
6645
6646     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
6647     set_float_rounding_mode(rmode, fp_status);
6648
6649     return prev_rmode;
6650 }
6651
6652 /* Half precision conversions.  */
6653 static float32 do_fcvt_f16_to_f32(uint32_t a, CPUARMState *env, float_status *s)
6654 {
6655     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6656     float32 r = float16_to_float32(make_float16(a), ieee, s);
6657     if (ieee) {
6658         return float32_maybe_silence_nan(r);
6659     }
6660     return r;
6661 }
6662
6663 static uint32_t do_fcvt_f32_to_f16(float32 a, CPUARMState *env, float_status *s)
6664 {
6665     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6666     float16 r = float32_to_float16(a, ieee, s);
6667     if (ieee) {
6668         r = float16_maybe_silence_nan(r);
6669     }
6670     return float16_val(r);
6671 }
6672
6673 float32 HELPER(neon_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
6674 {
6675     return do_fcvt_f16_to_f32(a, env, &env->vfp.standard_fp_status);
6676 }
6677
6678 uint32_t HELPER(neon_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
6679 {
6680     return do_fcvt_f32_to_f16(a, env, &env->vfp.standard_fp_status);
6681 }
6682
6683 float32 HELPER(vfp_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
6684 {
6685     return do_fcvt_f16_to_f32(a, env, &env->vfp.fp_status);
6686 }
6687
6688 uint32_t HELPER(vfp_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
6689 {
6690     return do_fcvt_f32_to_f16(a, env, &env->vfp.fp_status);
6691 }
6692
6693 float64 HELPER(vfp_fcvt_f16_to_f64)(uint32_t a, CPUARMState *env)
6694 {
6695     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6696     float64 r = float16_to_float64(make_float16(a), ieee, &env->vfp.fp_status);
6697     if (ieee) {
6698         return float64_maybe_silence_nan(r);
6699     }
6700     return r;
6701 }
6702
6703 uint32_t HELPER(vfp_fcvt_f64_to_f16)(float64 a, CPUARMState *env)
6704 {
6705     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6706     float16 r = float64_to_float16(a, ieee, &env->vfp.fp_status);
6707     if (ieee) {
6708         r = float16_maybe_silence_nan(r);
6709     }
6710     return float16_val(r);
6711 }
6712
6713 #define float32_two make_float32(0x40000000)
6714 #define float32_three make_float32(0x40400000)
6715 #define float32_one_point_five make_float32(0x3fc00000)
6716
6717 float32 HELPER(recps_f32)(float32 a, float32 b, CPUARMState *env)
6718 {
6719     float_status *s = &env->vfp.standard_fp_status;
6720     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
6721         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
6722         if (!(float32_is_zero(a) || float32_is_zero(b))) {
6723             float_raise(float_flag_input_denormal, s);
6724         }
6725         return float32_two;
6726     }
6727     return float32_sub(float32_two, float32_mul(a, b, s), s);
6728 }
6729
6730 float32 HELPER(rsqrts_f32)(float32 a, float32 b, CPUARMState *env)
6731 {
6732     float_status *s = &env->vfp.standard_fp_status;
6733     float32 product;
6734     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
6735         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
6736         if (!(float32_is_zero(a) || float32_is_zero(b))) {
6737             float_raise(float_flag_input_denormal, s);
6738         }
6739         return float32_one_point_five;
6740     }
6741     product = float32_mul(a, b, s);
6742     return float32_div(float32_sub(float32_three, product, s), float32_two, s);
6743 }
6744
6745 /* NEON helpers.  */
6746
6747 /* Constants 256 and 512 are used in some helpers; we avoid relying on
6748  * int->float conversions at run-time.  */
6749 #define float64_256 make_float64(0x4070000000000000LL)
6750 #define float64_512 make_float64(0x4080000000000000LL)
6751 #define float32_maxnorm make_float32(0x7f7fffff)
6752 #define float64_maxnorm make_float64(0x7fefffffffffffffLL)
6753
6754 /* Reciprocal functions
6755  *
6756  * The algorithm that must be used to calculate the estimate
6757  * is specified by the ARM ARM, see FPRecipEstimate()
6758  */
6759
6760 static float64 recip_estimate(float64 a, float_status *real_fp_status)
6761 {
6762     /* These calculations mustn't set any fp exception flags,
6763      * so we use a local copy of the fp_status.
6764      */
6765     float_status dummy_status = *real_fp_status;
6766     float_status *s = &dummy_status;
6767     /* q = (int)(a * 512.0) */
6768     float64 q = float64_mul(float64_512, a, s);
6769     int64_t q_int = float64_to_int64_round_to_zero(q, s);
6770
6771     /* r = 1.0 / (((double)q + 0.5) / 512.0) */
6772     q = int64_to_float64(q_int, s);
6773     q = float64_add(q, float64_half, s);
6774     q = float64_div(q, float64_512, s);
6775     q = float64_div(float64_one, q, s);
6776
6777     /* s = (int)(256.0 * r + 0.5) */
6778     q = float64_mul(q, float64_256, s);
6779     q = float64_add(q, float64_half, s);
6780     q_int = float64_to_int64_round_to_zero(q, s);
6781
6782     /* return (double)s / 256.0 */
6783     return float64_div(int64_to_float64(q_int, s), float64_256, s);
6784 }
6785
6786 /* Common wrapper to call recip_estimate */
6787 static float64 call_recip_estimate(float64 num, int off, float_status *fpst)
6788 {
6789     uint64_t val64 = float64_val(num);
6790     uint64_t frac = extract64(val64, 0, 52);
6791     int64_t exp = extract64(val64, 52, 11);
6792     uint64_t sbit;
6793     float64 scaled, estimate;
6794
6795     /* Generate the scaled number for the estimate function */
6796     if (exp == 0) {
6797         if (extract64(frac, 51, 1) == 0) {
6798             exp = -1;
6799             frac = extract64(frac, 0, 50) << 2;
6800         } else {
6801             frac = extract64(frac, 0, 51) << 1;
6802         }
6803     }
6804
6805     /* scaled = '0' : '01111111110' : fraction<51:44> : Zeros(44); */
6806     scaled = make_float64((0x3feULL << 52)
6807                           | extract64(frac, 44, 8) << 44);
6808
6809     estimate = recip_estimate(scaled, fpst);
6810
6811     /* Build new result */
6812     val64 = float64_val(estimate);
6813     sbit = 0x8000000000000000ULL & val64;
6814     exp = off - exp;
6815     frac = extract64(val64, 0, 52);
6816
6817     if (exp == 0) {
6818         frac = 1ULL << 51 | extract64(frac, 1, 51);
6819     } else if (exp == -1) {
6820         frac = 1ULL << 50 | extract64(frac, 2, 50);
6821         exp = 0;
6822     }
6823
6824     return make_float64(sbit | (exp << 52) | frac);
6825 }
6826
6827 static bool round_to_inf(float_status *fpst, bool sign_bit)
6828 {
6829     switch (fpst->float_rounding_mode) {
6830     case float_round_nearest_even: /* Round to Nearest */
6831         return true;
6832     case float_round_up: /* Round to +Inf */
6833         return !sign_bit;
6834     case float_round_down: /* Round to -Inf */
6835         return sign_bit;
6836     case float_round_to_zero: /* Round to Zero */
6837         return false;
6838     }
6839
6840     g_assert_not_reached();
6841 }
6842
6843 float32 HELPER(recpe_f32)(float32 input, void *fpstp)
6844 {
6845     float_status *fpst = fpstp;
6846     float32 f32 = float32_squash_input_denormal(input, fpst);
6847     uint32_t f32_val = float32_val(f32);
6848     uint32_t f32_sbit = 0x80000000ULL & f32_val;
6849     int32_t f32_exp = extract32(f32_val, 23, 8);
6850     uint32_t f32_frac = extract32(f32_val, 0, 23);
6851     float64 f64, r64;
6852     uint64_t r64_val;
6853     int64_t r64_exp;
6854     uint64_t r64_frac;
6855
6856     if (float32_is_any_nan(f32)) {
6857         float32 nan = f32;
6858         if (float32_is_signaling_nan(f32)) {
6859             float_raise(float_flag_invalid, fpst);
6860             nan = float32_maybe_silence_nan(f32);
6861         }
6862         if (fpst->default_nan_mode) {
6863             nan =  float32_default_nan;
6864         }
6865         return nan;
6866     } else if (float32_is_infinity(f32)) {
6867         return float32_set_sign(float32_zero, float32_is_neg(f32));
6868     } else if (float32_is_zero(f32)) {
6869         float_raise(float_flag_divbyzero, fpst);
6870         return float32_set_sign(float32_infinity, float32_is_neg(f32));
6871     } else if ((f32_val & ~(1ULL << 31)) < (1ULL << 21)) {
6872         /* Abs(value) < 2.0^-128 */
6873         float_raise(float_flag_overflow | float_flag_inexact, fpst);
6874         if (round_to_inf(fpst, f32_sbit)) {
6875             return float32_set_sign(float32_infinity, float32_is_neg(f32));
6876         } else {
6877             return float32_set_sign(float32_maxnorm, float32_is_neg(f32));
6878         }
6879     } else if (f32_exp >= 253 && fpst->flush_to_zero) {
6880         float_raise(float_flag_underflow, fpst);
6881         return float32_set_sign(float32_zero, float32_is_neg(f32));
6882     }
6883
6884
6885     f64 = make_float64(((int64_t)(f32_exp) << 52) | (int64_t)(f32_frac) << 29);
6886     r64 = call_recip_estimate(f64, 253, fpst);
6887     r64_val = float64_val(r64);
6888     r64_exp = extract64(r64_val, 52, 11);
6889     r64_frac = extract64(r64_val, 0, 52);
6890
6891     /* result = sign : result_exp<7:0> : fraction<51:29>; */
6892     return make_float32(f32_sbit |
6893                         (r64_exp & 0xff) << 23 |
6894                         extract64(r64_frac, 29, 24));
6895 }
6896
6897 float64 HELPER(recpe_f64)(float64 input, void *fpstp)
6898 {
6899     float_status *fpst = fpstp;
6900     float64 f64 = float64_squash_input_denormal(input, fpst);
6901     uint64_t f64_val = float64_val(f64);
6902     uint64_t f64_sbit = 0x8000000000000000ULL & f64_val;
6903     int64_t f64_exp = extract64(f64_val, 52, 11);
6904     float64 r64;
6905     uint64_t r64_val;
6906     int64_t r64_exp;
6907     uint64_t r64_frac;
6908
6909     /* Deal with any special cases */
6910     if (float64_is_any_nan(f64)) {
6911         float64 nan = f64;
6912         if (float64_is_signaling_nan(f64)) {
6913             float_raise(float_flag_invalid, fpst);
6914             nan = float64_maybe_silence_nan(f64);
6915         }
6916         if (fpst->default_nan_mode) {
6917             nan =  float64_default_nan;
6918         }
6919         return nan;
6920     } else if (float64_is_infinity(f64)) {
6921         return float64_set_sign(float64_zero, float64_is_neg(f64));
6922     } else if (float64_is_zero(f64)) {
6923         float_raise(float_flag_divbyzero, fpst);
6924         return float64_set_sign(float64_infinity, float64_is_neg(f64));
6925     } else if ((f64_val & ~(1ULL << 63)) < (1ULL << 50)) {
6926         /* Abs(value) < 2.0^-1024 */
6927         float_raise(float_flag_overflow | float_flag_inexact, fpst);
6928         if (round_to_inf(fpst, f64_sbit)) {
6929             return float64_set_sign(float64_infinity, float64_is_neg(f64));
6930         } else {
6931             return float64_set_sign(float64_maxnorm, float64_is_neg(f64));
6932         }
6933     } else if (f64_exp >= 2045 && fpst->flush_to_zero) {
6934         float_raise(float_flag_underflow, fpst);
6935         return float64_set_sign(float64_zero, float64_is_neg(f64));
6936     }
6937
6938     r64 = call_recip_estimate(f64, 2045, fpst);
6939     r64_val = float64_val(r64);
6940     r64_exp = extract64(r64_val, 52, 11);
6941     r64_frac = extract64(r64_val, 0, 52);
6942
6943     /* result = sign : result_exp<10:0> : fraction<51:0> */
6944     return make_float64(f64_sbit |
6945                         ((r64_exp & 0x7ff) << 52) |
6946                         r64_frac);
6947 }
6948
6949 /* The algorithm that must be used to calculate the estimate
6950  * is specified by the ARM ARM.
6951  */
6952 static float64 recip_sqrt_estimate(float64 a, float_status *real_fp_status)
6953 {
6954     /* These calculations mustn't set any fp exception flags,
6955      * so we use a local copy of the fp_status.
6956      */
6957     float_status dummy_status = *real_fp_status;
6958     float_status *s = &dummy_status;
6959     float64 q;
6960     int64_t q_int;
6961
6962     if (float64_lt(a, float64_half, s)) {
6963         /* range 0.25 <= a < 0.5 */
6964
6965         /* a in units of 1/512 rounded down */
6966         /* q0 = (int)(a * 512.0);  */
6967         q = float64_mul(float64_512, a, s);
6968         q_int = float64_to_int64_round_to_zero(q, s);
6969
6970         /* reciprocal root r */
6971         /* r = 1.0 / sqrt(((double)q0 + 0.5) / 512.0);  */
6972         q = int64_to_float64(q_int, s);
6973         q = float64_add(q, float64_half, s);
6974         q = float64_div(q, float64_512, s);
6975         q = float64_sqrt(q, s);
6976         q = float64_div(float64_one, q, s);
6977     } else {
6978         /* range 0.5 <= a < 1.0 */
6979
6980         /* a in units of 1/256 rounded down */
6981         /* q1 = (int)(a * 256.0); */
6982         q = float64_mul(float64_256, a, s);
6983         int64_t q_int = float64_to_int64_round_to_zero(q, s);
6984
6985         /* reciprocal root r */
6986         /* r = 1.0 /sqrt(((double)q1 + 0.5) / 256); */
6987         q = int64_to_float64(q_int, s);
6988         q = float64_add(q, float64_half, s);
6989         q = float64_div(q, float64_256, s);
6990         q = float64_sqrt(q, s);
6991         q = float64_div(float64_one, q, s);
6992     }
6993     /* r in units of 1/256 rounded to nearest */
6994     /* s = (int)(256.0 * r + 0.5); */
6995
6996     q = float64_mul(q, float64_256,s );
6997     q = float64_add(q, float64_half, s);
6998     q_int = float64_to_int64_round_to_zero(q, s);
6999
7000     /* return (double)s / 256.0;*/
7001     return float64_div(int64_to_float64(q_int, s), float64_256, s);
7002 }
7003
7004 float32 HELPER(rsqrte_f32)(float32 input, void *fpstp)
7005 {
7006     float_status *s = fpstp;
7007     float32 f32 = float32_squash_input_denormal(input, s);
7008     uint32_t val = float32_val(f32);
7009     uint32_t f32_sbit = 0x80000000 & val;
7010     int32_t f32_exp = extract32(val, 23, 8);
7011     uint32_t f32_frac = extract32(val, 0, 23);
7012     uint64_t f64_frac;
7013     uint64_t val64;
7014     int result_exp;
7015     float64 f64;
7016
7017     if (float32_is_any_nan(f32)) {
7018         float32 nan = f32;
7019         if (float32_is_signaling_nan(f32)) {
7020             float_raise(float_flag_invalid, s);
7021             nan = float32_maybe_silence_nan(f32);
7022         }
7023         if (s->default_nan_mode) {
7024             nan =  float32_default_nan;
7025         }
7026         return nan;
7027     } else if (float32_is_zero(f32)) {
7028         float_raise(float_flag_divbyzero, s);
7029         return float32_set_sign(float32_infinity, float32_is_neg(f32));
7030     } else if (float32_is_neg(f32)) {
7031         float_raise(float_flag_invalid, s);
7032         return float32_default_nan;
7033     } else if (float32_is_infinity(f32)) {
7034         return float32_zero;
7035     }
7036
7037     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
7038      * preserving the parity of the exponent.  */
7039
7040     f64_frac = ((uint64_t) f32_frac) << 29;
7041     if (f32_exp == 0) {
7042         while (extract64(f64_frac, 51, 1) == 0) {
7043             f64_frac = f64_frac << 1;
7044             f32_exp = f32_exp-1;
7045         }
7046         f64_frac = extract64(f64_frac, 0, 51) << 1;
7047     }
7048
7049     if (extract64(f32_exp, 0, 1) == 0) {
7050         f64 = make_float64(((uint64_t) f32_sbit) << 32
7051                            | (0x3feULL << 52)
7052                            | f64_frac);
7053     } else {
7054         f64 = make_float64(((uint64_t) f32_sbit) << 32
7055                            | (0x3fdULL << 52)
7056                            | f64_frac);
7057     }
7058
7059     result_exp = (380 - f32_exp) / 2;
7060
7061     f64 = recip_sqrt_estimate(f64, s);
7062
7063     val64 = float64_val(f64);
7064
7065     val = ((result_exp & 0xff) << 23)
7066         | ((val64 >> 29)  & 0x7fffff);
7067     return make_float32(val);
7068 }
7069
7070 float64 HELPER(rsqrte_f64)(float64 input, void *fpstp)
7071 {
7072     float_status *s = fpstp;
7073     float64 f64 = float64_squash_input_denormal(input, s);
7074     uint64_t val = float64_val(f64);
7075     uint64_t f64_sbit = 0x8000000000000000ULL & val;
7076     int64_t f64_exp = extract64(val, 52, 11);
7077     uint64_t f64_frac = extract64(val, 0, 52);
7078     int64_t result_exp;
7079     uint64_t result_frac;
7080
7081     if (float64_is_any_nan(f64)) {
7082         float64 nan = f64;
7083         if (float64_is_signaling_nan(f64)) {
7084             float_raise(float_flag_invalid, s);
7085             nan = float64_maybe_silence_nan(f64);
7086         }
7087         if (s->default_nan_mode) {
7088             nan =  float64_default_nan;
7089         }
7090         return nan;
7091     } else if (float64_is_zero(f64)) {
7092         float_raise(float_flag_divbyzero, s);
7093         return float64_set_sign(float64_infinity, float64_is_neg(f64));
7094     } else if (float64_is_neg(f64)) {
7095         float_raise(float_flag_invalid, s);
7096         return float64_default_nan;
7097     } else if (float64_is_infinity(f64)) {
7098         return float64_zero;
7099     }
7100
7101     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
7102      * preserving the parity of the exponent.  */
7103
7104     if (f64_exp == 0) {
7105         while (extract64(f64_frac, 51, 1) == 0) {
7106             f64_frac = f64_frac << 1;
7107             f64_exp = f64_exp - 1;
7108         }
7109         f64_frac = extract64(f64_frac, 0, 51) << 1;
7110     }
7111
7112     if (extract64(f64_exp, 0, 1) == 0) {
7113         f64 = make_float64(f64_sbit
7114                            | (0x3feULL << 52)
7115                            | f64_frac);
7116     } else {
7117         f64 = make_float64(f64_sbit
7118                            | (0x3fdULL << 52)
7119                            | f64_frac);
7120     }
7121
7122     result_exp = (3068 - f64_exp) / 2;
7123
7124     f64 = recip_sqrt_estimate(f64, s);
7125
7126     result_frac = extract64(float64_val(f64), 0, 52);
7127
7128     return make_float64(f64_sbit |
7129                         ((result_exp & 0x7ff) << 52) |
7130                         result_frac);
7131 }
7132
7133 uint32_t HELPER(recpe_u32)(uint32_t a, void *fpstp)
7134 {
7135     float_status *s = fpstp;
7136     float64 f64;
7137
7138     if ((a & 0x80000000) == 0) {
7139         return 0xffffffff;
7140     }
7141
7142     f64 = make_float64((0x3feULL << 52)
7143                        | ((int64_t)(a & 0x7fffffff) << 21));
7144
7145     f64 = recip_estimate(f64, s);
7146
7147     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
7148 }
7149
7150 uint32_t HELPER(rsqrte_u32)(uint32_t a, void *fpstp)
7151 {
7152     float_status *fpst = fpstp;
7153     float64 f64;
7154
7155     if ((a & 0xc0000000) == 0) {
7156         return 0xffffffff;
7157     }
7158
7159     if (a & 0x80000000) {
7160         f64 = make_float64((0x3feULL << 52)
7161                            | ((uint64_t)(a & 0x7fffffff) << 21));
7162     } else { /* bits 31-30 == '01' */
7163         f64 = make_float64((0x3fdULL << 52)
7164                            | ((uint64_t)(a & 0x3fffffff) << 22));
7165     }
7166
7167     f64 = recip_sqrt_estimate(f64, fpst);
7168
7169     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
7170 }
7171
7172 /* VFPv4 fused multiply-accumulate */
7173 float32 VFP_HELPER(muladd, s)(float32 a, float32 b, float32 c, void *fpstp)
7174 {
7175     float_status *fpst = fpstp;
7176     return float32_muladd(a, b, c, 0, fpst);
7177 }
7178
7179 float64 VFP_HELPER(muladd, d)(float64 a, float64 b, float64 c, void *fpstp)
7180 {
7181     float_status *fpst = fpstp;
7182     return float64_muladd(a, b, c, 0, fpst);
7183 }
7184
7185 /* ARMv8 round to integral */
7186 float32 HELPER(rints_exact)(float32 x, void *fp_status)
7187 {
7188     return float32_round_to_int(x, fp_status);
7189 }
7190
7191 float64 HELPER(rintd_exact)(float64 x, void *fp_status)
7192 {
7193     return float64_round_to_int(x, fp_status);
7194 }
7195
7196 float32 HELPER(rints)(float32 x, void *fp_status)
7197 {
7198     int old_flags = get_float_exception_flags(fp_status), new_flags;
7199     float32 ret;
7200
7201     ret = float32_round_to_int(x, fp_status);
7202
7203     /* Suppress any inexact exceptions the conversion produced */
7204     if (!(old_flags & float_flag_inexact)) {
7205         new_flags = get_float_exception_flags(fp_status);
7206         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
7207     }
7208
7209     return ret;
7210 }
7211
7212 float64 HELPER(rintd)(float64 x, void *fp_status)
7213 {
7214     int old_flags = get_float_exception_flags(fp_status), new_flags;
7215     float64 ret;
7216
7217     ret = float64_round_to_int(x, fp_status);
7218
7219     new_flags = get_float_exception_flags(fp_status);
7220
7221     /* Suppress any inexact exceptions the conversion produced */
7222     if (!(old_flags & float_flag_inexact)) {
7223         new_flags = get_float_exception_flags(fp_status);
7224         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
7225     }
7226
7227     return ret;
7228 }
7229
7230 /* Convert ARM rounding mode to softfloat */
7231 int arm_rmode_to_sf(int rmode)
7232 {
7233     switch (rmode) {
7234     case FPROUNDING_TIEAWAY:
7235         rmode = float_round_ties_away;
7236         break;
7237     case FPROUNDING_ODD:
7238         /* FIXME: add support for TIEAWAY and ODD */
7239         qemu_log_mask(LOG_UNIMP, "arm: unimplemented rounding mode: %d\n",
7240                       rmode);
7241     case FPROUNDING_TIEEVEN:
7242     default:
7243         rmode = float_round_nearest_even;
7244         break;
7245     case FPROUNDING_POSINF:
7246         rmode = float_round_up;
7247         break;
7248     case FPROUNDING_NEGINF:
7249         rmode = float_round_down;
7250         break;
7251     case FPROUNDING_ZERO:
7252         rmode = float_round_to_zero;
7253         break;
7254     }
7255     return rmode;
7256 }
7257
7258 /* CRC helpers.
7259  * The upper bytes of val (above the number specified by 'bytes') must have
7260  * been zeroed out by the caller.
7261  */
7262 uint32_t HELPER(crc32)(uint32_t acc, uint32_t val, uint32_t bytes)
7263 {
7264     uint8_t buf[4];
7265
7266     stl_le_p(buf, val);
7267
7268     /* zlib crc32 converts the accumulator and output to one's complement.  */
7269     return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
7270 }
7271
7272 uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
7273 {
7274     uint8_t buf[4];
7275
7276     stl_le_p(buf, val);
7277
7278     /* Linux crc32c converts the output to one's complement.  */
7279     return crc32c(acc, buf, bytes) ^ 0xffffffff;
7280 }