target-arm/helper.c

   1 #include "qemu/osdep.h"
   2 #include "cpu.h"
   3 #include "internals.h"
   4 #include "exec/gdbstub.h"
   5 #include "exec/helper-proto.h"
   6 #include "qemu/host-utils.h"
   7 #include "sysemu/arch_init.h"
   8 #include "sysemu/sysemu.h"
   9 #include "qemu/bitops.h"
  10 #include "qemu/crc32c.h"
  11 #include "exec/exec-all.h"
  12 #include "exec/cpu_ldst.h"
  13 #include "arm_ldst.h"
  14 #include <zlib.h> /* For crc32 */
  15 #include "exec/semihost.h"
  16 #include "sysemu/kvm.h"
  17
  18 #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */
  19
  20 #ifndef CONFIG_USER_ONLY
  21 static bool get_phys_addr(CPUARMState *env, target_ulong address,
  22                           int access_type, ARMMMUIdx mmu_idx,
  23                           hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
  24                           target_ulong *page_size, uint32_t *fsr,
  25                           ARMMMUFaultInfo *fi);
  26
  27 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
  28                                int access_type, ARMMMUIdx mmu_idx,
  29                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
  30                                target_ulong *page_size_ptr, uint32_t *fsr,
  31                                ARMMMUFaultInfo *fi);
  32
  33 /* Definitions for the PMCCNTR and PMCR registers */
  34 #define PMCRD   0x8
  35 #define PMCRC   0x4
  36 #define PMCRE   0x1
  37 #endif
  38
  39 static int vfp_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  40 {
  41     int nregs;
  42
  43     /* VFP data registers are always little-endian.  */
  44     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  45     if (reg < nregs) {
  46         stfq_le_p(buf, env->vfp.regs[reg]);
  47         return 8;
  48     }
  49     if (arm_feature(env, ARM_FEATURE_NEON)) {
  50         /* Aliases for Q regs.  */
  51         nregs += 16;
  52         if (reg < nregs) {
  53             stfq_le_p(buf, env->vfp.regs[(reg - 32) * 2]);
  54             stfq_le_p(buf + 8, env->vfp.regs[(reg - 32) * 2 + 1]);
  55             return 16;
  56         }
  57     }
  58     switch (reg - nregs) {
  59     case 0: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSID]); return 4;
  60     case 1: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSCR]); return 4;
  61     case 2: stl_p(buf, env->vfp.xregs[ARM_VFP_FPEXC]); return 4;
  62     }
  63     return 0;
  64 }
  65
  66 static int vfp_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
  67 {
  68     int nregs;
  69
  70     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  71     if (reg < nregs) {
  72         env->vfp.regs[reg] = ldfq_le_p(buf);
  73         return 8;
  74     }
  75     if (arm_feature(env, ARM_FEATURE_NEON)) {
  76         nregs += 16;
  77         if (reg < nregs) {
  78             env->vfp.regs[(reg - 32) * 2] = ldfq_le_p(buf);
  79             env->vfp.regs[(reg - 32) * 2 + 1] = ldfq_le_p(buf + 8);
  80             return 16;
  81         }
  82     }
  83     switch (reg - nregs) {
  84     case 0: env->vfp.xregs[ARM_VFP_FPSID] = ldl_p(buf); return 4;
  85     case 1: env->vfp.xregs[ARM_VFP_FPSCR] = ldl_p(buf); return 4;
  86     case 2: env->vfp.xregs[ARM_VFP_FPEXC] = ldl_p(buf) & (1 << 30); return 4;
  87     }
  88     return 0;
  89 }
  90
  91 static int aarch64_fpu_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  92 {
  93     switch (reg) {
  94     case 0 ... 31:
  95         /* 128 bit FP register */
  96         stfq_le_p(buf, env->vfp.regs[reg * 2]);
  97         stfq_le_p(buf + 8, env->vfp.regs[reg * 2 + 1]);
  98         return 16;
  99     case 32:
 100         /* FPSR */
 101         stl_p(buf, vfp_get_fpsr(env));
 102         return 4;
 103     case 33:
 104         /* FPCR */
 105         stl_p(buf, vfp_get_fpcr(env));
 106         return 4;
 107     default:
 108         return 0;
 109     }
 110 }
 111
 112 static int aarch64_fpu_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
 113 {
 114     switch (reg) {
 115     case 0 ... 31:
 116         /* 128 bit FP register */
 117         env->vfp.regs[reg * 2] = ldfq_le_p(buf);
 118         env->vfp.regs[reg * 2 + 1] = ldfq_le_p(buf + 8);
 119         return 16;
 120     case 32:
 121         /* FPSR */
 122         vfp_set_fpsr(env, ldl_p(buf));
 123         return 4;
 124     case 33:
 125         /* FPCR */
 126         vfp_set_fpcr(env, ldl_p(buf));
 127         return 4;
 128     default:
 129         return 0;
 130     }
 131 }
 132
 133 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
 134 {
 135     assert(ri->fieldoffset);
 136     if (cpreg_field_is_64bit(ri)) {
 137         return CPREG_FIELD64(env, ri);
 138     } else {
 139         return CPREG_FIELD32(env, ri);
 140     }
 141 }
 142
 143 static void raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
 144                       uint64_t value)
 145 {
 146     assert(ri->fieldoffset);
 147     if (cpreg_field_is_64bit(ri)) {
 148         CPREG_FIELD64(env, ri) = value;
 149     } else {
 150         CPREG_FIELD32(env, ri) = value;
 151     }
 152 }
 153
 154 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 155 {
 156     return (char *)env + ri->fieldoffset;
 157 }
 158
 159 uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 160 {
 161     /* Raw read of a coprocessor register (as needed for migration, etc). */
 162     if (ri->type & ARM_CP_CONST) {
 163         return ri->resetvalue;
 164     } else if (ri->raw_readfn) {
 165         return ri->raw_readfn(env, ri);
 166     } else if (ri->readfn) {
 167         return ri->readfn(env, ri);
 168     } else {
 169         return raw_read(env, ri);
 170     }
 171 }
 172
 173 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
 174                              uint64_t v)
 175 {
 176     /* Raw write of a coprocessor register (as needed for migration, etc).
 177      * Note that constant registers are treated as write-ignored; the
 178      * caller should check for success by whether a readback gives the
 179      * value written.
 180      */
 181     if (ri->type & ARM_CP_CONST) {
 182         return;
 183     } else if (ri->raw_writefn) {
 184         ri->raw_writefn(env, ri, v);
 185     } else if (ri->writefn) {
 186         ri->writefn(env, ri, v);
 187     } else {
 188         raw_write(env, ri, v);
 189     }
 190 }
 191
 192 static bool raw_accessors_invalid(const ARMCPRegInfo *ri)
 193 {
 194    /* Return true if the regdef would cause an assertion if you called
 195     * read_raw_cp_reg() or write_raw_cp_reg() on it (ie if it is a
 196     * program bug for it not to have the NO_RAW flag).
 197     * NB that returning false here doesn't necessarily mean that calling
 198     * read/write_raw_cp_reg() is safe, because we can't distinguish "has
 199     * read/write access functions which are safe for raw use" from "has
 200     * read/write access functions which have side effects but has forgotten
 201     * to provide raw access functions".
 202     * The tests here line up with the conditions in read/write_raw_cp_reg()
 203     * and assertions in raw_read()/raw_write().
 204     */
 205     if ((ri->type & ARM_CP_CONST) ||
 206         ri->fieldoffset ||
 207         ((ri->raw_writefn || ri->writefn) && (ri->raw_readfn || ri->readfn))) {
 208         return false;
 209     }
 210     return true;
 211 }
 212
 213 bool write_cpustate_to_list(ARMCPU *cpu)
 214 {
 215     /* Write the coprocessor state from cpu->env to the (index,value) list. */
 216     int i;
 217     bool ok = true;
 218
 219     for (i = 0; i < cpu->cpreg_array_len; i++) {
 220         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 221         const ARMCPRegInfo *ri;
 222
 223         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 224         if (!ri) {
 225             ok = false;
 226             continue;
 227         }
 228         if (ri->type & ARM_CP_NO_RAW) {
 229             continue;
 230         }
 231         cpu->cpreg_values[i] = read_raw_cp_reg(&cpu->env, ri);
 232     }
 233     return ok;
 234 }
 235
 236 bool write_list_to_cpustate(ARMCPU *cpu)
 237 {
 238     int i;
 239     bool ok = true;
 240
 241     for (i = 0; i < cpu->cpreg_array_len; i++) {
 242         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 243         uint64_t v = cpu->cpreg_values[i];
 244         const ARMCPRegInfo *ri;
 245
 246         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 247         if (!ri) {
 248             ok = false;
 249             continue;
 250         }
 251         if (ri->type & ARM_CP_NO_RAW) {
 252             continue;
 253         }
 254         /* Write value and confirm it reads back as written
 255          * (to catch read-only registers and partially read-only
 256          * registers where the incoming migration value doesn't match)
 257          */
 258         write_raw_cp_reg(&cpu->env, ri, v);
 259         if (read_raw_cp_reg(&cpu->env, ri) != v) {
 260             ok = false;
 261         }
 262     }
 263     return ok;
 264 }
 265
 266 static void add_cpreg_to_list(gpointer key, gpointer opaque)
 267 {
 268     ARMCPU *cpu = opaque;
 269     uint64_t regidx;
 270     const ARMCPRegInfo *ri;
 271
 272     regidx = *(uint32_t *)key;
 273     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 274
 275     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 276         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
 277         /* The value array need not be initialized at this point */
 278         cpu->cpreg_array_len++;
 279     }
 280 }
 281
 282 static void count_cpreg(gpointer key, gpointer opaque)
 283 {
 284     ARMCPU *cpu = opaque;
 285     uint64_t regidx;
 286     const ARMCPRegInfo *ri;
 287
 288     regidx = *(uint32_t *)key;
 289     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 290
 291     if (!(ri->type & (ARM_CP_NO_RAW|ARM_CP_ALIAS))) {
 292         cpu->cpreg_array_len++;
 293     }
 294 }
 295
 296 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
 297 {
 298     uint64_t aidx = cpreg_to_kvm_id(*(uint32_t *)a);
 299     uint64_t bidx = cpreg_to_kvm_id(*(uint32_t *)b);
 300
 301     if (aidx > bidx) {
 302         return 1;
 303     }
 304     if (aidx < bidx) {
 305         return -1;
 306     }
 307     return 0;
 308 }
 309
 310 void init_cpreg_list(ARMCPU *cpu)
 311 {
 312     /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
 313      * Note that we require cpreg_tuples[] to be sorted by key ID.
 314      */
 315     GList *keys;
 316     int arraylen;
 317
 318     keys = g_hash_table_get_keys(cpu->cp_regs);
 319     keys = g_list_sort(keys, cpreg_key_compare);
 320
 321     cpu->cpreg_array_len = 0;
 322
 323     g_list_foreach(keys, count_cpreg, cpu);
 324
 325     arraylen = cpu->cpreg_array_len;
 326     cpu->cpreg_indexes = g_new(uint64_t, arraylen);
 327     cpu->cpreg_values = g_new(uint64_t, arraylen);
 328     cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
 329     cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
 330     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
 331     cpu->cpreg_array_len = 0;
 332
 333     g_list_foreach(keys, add_cpreg_to_list, cpu);
 334
 335     assert(cpu->cpreg_array_len == arraylen);
 336
 337     g_list_free(keys);
 338 }
 339
 340 /*
 341  * Some registers are not accessible if EL3.NS=0 and EL3 is using AArch32 but
 342  * they are accessible when EL3 is using AArch64 regardless of EL3.NS.
 343  *
 344  * access_el3_aa32ns: Used to check AArch32 register views.
 345  * access_el3_aa32ns_aa64any: Used to check both AArch32/64 register views.
 346  */
 347 static CPAccessResult access_el3_aa32ns(CPUARMState *env,
 348                                         const ARMCPRegInfo *ri,
 349                                         bool isread)
 350 {
 351     bool secure = arm_is_secure_below_el3(env);
 352
 353     assert(!arm_el_is_aa64(env, 3));
 354     if (secure) {
 355         return CP_ACCESS_TRAP_UNCATEGORIZED;
 356     }
 357     return CP_ACCESS_OK;
 358 }
 359
 360 static CPAccessResult access_el3_aa32ns_aa64any(CPUARMState *env,
 361                                                 const ARMCPRegInfo *ri,
 362                                                 bool isread)
 363 {
 364     if (!arm_el_is_aa64(env, 3)) {
 365         return access_el3_aa32ns(env, ri, isread);
 366     }
 367     return CP_ACCESS_OK;
 368 }
 369
 370 /* Some secure-only AArch32 registers trap to EL3 if used from
 371  * Secure EL1 (but are just ordinary UNDEF in other non-EL3 contexts).
 372  * Note that an access from Secure EL1 can only happen if EL3 is AArch64.
 373  * We assume that the .access field is set to PL1_RW.
 374  */
 375 static CPAccessResult access_trap_aa32s_el1(CPUARMState *env,
 376                                             const ARMCPRegInfo *ri,
 377                                             bool isread)
 378 {
 379     if (arm_current_el(env) == 3) {
 380         return CP_ACCESS_OK;
 381     }
 382     if (arm_is_secure_below_el3(env)) {
 383         return CP_ACCESS_TRAP_EL3;
 384     }
 385     /* This will be EL1 NS and EL2 NS, which just UNDEF */
 386     return CP_ACCESS_TRAP_UNCATEGORIZED;
 387 }
 388
 389 /* Check for traps to "powerdown debug" registers, which are controlled
 390  * by MDCR.TDOSA
 391  */
 392 static CPAccessResult access_tdosa(CPUARMState *env, const ARMCPRegInfo *ri,
 393                                    bool isread)
 394 {
 395     int el = arm_current_el(env);
 396
 397     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDOSA)
 398         && !arm_is_secure_below_el3(env)) {
 399         return CP_ACCESS_TRAP_EL2;
 400     }
 401     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDOSA)) {
 402         return CP_ACCESS_TRAP_EL3;
 403     }
 404     return CP_ACCESS_OK;
 405 }
 406
 407 /* Check for traps to "debug ROM" registers, which are controlled
 408  * by MDCR_EL2.TDRA for EL2 but by the more general MDCR_EL3.TDA for EL3.
 409  */
 410 static CPAccessResult access_tdra(CPUARMState *env, const ARMCPRegInfo *ri,
 411                                   bool isread)
 412 {
 413     int el = arm_current_el(env);
 414
 415     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDRA)
 416         && !arm_is_secure_below_el3(env)) {
 417         return CP_ACCESS_TRAP_EL2;
 418     }
 419     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
 420         return CP_ACCESS_TRAP_EL3;
 421     }
 422     return CP_ACCESS_OK;
 423 }
 424
 425 /* Check for traps to general debug registers, which are controlled
 426  * by MDCR_EL2.TDA for EL2 and MDCR_EL3.TDA for EL3.
 427  */
 428 static CPAccessResult access_tda(CPUARMState *env, const ARMCPRegInfo *ri,
 429                                   bool isread)
 430 {
 431     int el = arm_current_el(env);
 432
 433     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TDA)
 434         && !arm_is_secure_below_el3(env)) {
 435         return CP_ACCESS_TRAP_EL2;
 436     }
 437     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TDA)) {
 438         return CP_ACCESS_TRAP_EL3;
 439     }
 440     return CP_ACCESS_OK;
 441 }
 442
 443 /* Check for traps to performance monitor registers, which are controlled
 444  * by MDCR_EL2.TPM for EL2 and MDCR_EL3.TPM for EL3.
 445  */
 446 static CPAccessResult access_tpm(CPUARMState *env, const ARMCPRegInfo *ri,
 447                                  bool isread)
 448 {
 449     int el = arm_current_el(env);
 450
 451     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
 452         && !arm_is_secure_below_el3(env)) {
 453         return CP_ACCESS_TRAP_EL2;
 454     }
 455     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
 456         return CP_ACCESS_TRAP_EL3;
 457     }
 458     return CP_ACCESS_OK;
 459 }
 460
 461 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 462 {
 463     ARMCPU *cpu = arm_env_get_cpu(env);
 464
 465     raw_write(env, ri, value);
 466     tlb_flush(CPU(cpu), 1); /* Flush TLB as domain not tracked in TLB */
 467 }
 468
 469 static void fcse_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 470 {
 471     ARMCPU *cpu = arm_env_get_cpu(env);
 472
 473     if (raw_read(env, ri) != value) {
 474         /* Unlike real hardware the qemu TLB uses virtual addresses,
 475          * not modified virtual addresses, so this causes a TLB flush.
 476          */
 477         tlb_flush(CPU(cpu), 1);
 478         raw_write(env, ri, value);
 479     }
 480 }
 481
 482 static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 483                              uint64_t value)
 484 {
 485     ARMCPU *cpu = arm_env_get_cpu(env);
 486
 487     if (raw_read(env, ri) != value && !arm_feature(env, ARM_FEATURE_MPU)
 488         && !extended_addresses_enabled(env)) {
 489         /* For VMSA (when not using the LPAE long descriptor page table
 490          * format) this register includes the ASID, so do a TLB flush.
 491          * For PMSA it is purely a process ID and no action is needed.
 492          */
 493         tlb_flush(CPU(cpu), 1);
 494     }
 495     raw_write(env, ri, value);
 496 }
 497
 498 static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
 499                           uint64_t value)
 500 {
 501     /* Invalidate all (TLBIALL) */
 502     ARMCPU *cpu = arm_env_get_cpu(env);
 503
 504     tlb_flush(CPU(cpu), 1);
 505 }
 506
 507 static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri,
 508                           uint64_t value)
 509 {
 510     /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */
 511     ARMCPU *cpu = arm_env_get_cpu(env);
 512
 513     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 514 }
 515
 516 static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri,
 517                            uint64_t value)
 518 {
 519     /* Invalidate by ASID (TLBIASID) */
 520     ARMCPU *cpu = arm_env_get_cpu(env);
 521
 522     tlb_flush(CPU(cpu), value == 0);
 523 }
 524
 525 static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
 526                            uint64_t value)
 527 {
 528     /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */
 529     ARMCPU *cpu = arm_env_get_cpu(env);
 530
 531     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 532 }
 533
 534 /* IS variants of TLB operations must affect all cores */
 535 static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 536                              uint64_t value)
 537 {
 538     CPUState *other_cs;
 539
 540     CPU_FOREACH(other_cs) {
 541         tlb_flush(other_cs, 1);
 542     }
 543 }
 544
 545 static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 546                              uint64_t value)
 547 {
 548     CPUState *other_cs;
 549
 550     CPU_FOREACH(other_cs) {
 551         tlb_flush(other_cs, value == 0);
 552     }
 553 }
 554
 555 static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 556                              uint64_t value)
 557 {
 558     CPUState *other_cs;
 559
 560     CPU_FOREACH(other_cs) {
 561         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 562     }
 563 }
 564
 565 static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 566                              uint64_t value)
 567 {
 568     CPUState *other_cs;
 569
 570     CPU_FOREACH(other_cs) {
 571         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 572     }
 573 }
 574
 575 static void tlbiall_nsnh_write(CPUARMState *env, const ARMCPRegInfo *ri,
 576                                uint64_t value)
 577 {
 578     CPUState *cs = ENV_GET_CPU(env);
 579
 580     tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
 581                         ARMMMUIdx_S2NS, -1);
 582 }
 583
 584 static void tlbiall_nsnh_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 585                                   uint64_t value)
 586 {
 587     CPUState *other_cs;
 588
 589     CPU_FOREACH(other_cs) {
 590         tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
 591                             ARMMMUIdx_S12NSE0, ARMMMUIdx_S2NS, -1);
 592     }
 593 }
 594
 595 static void tlbiipas2_write(CPUARMState *env, const ARMCPRegInfo *ri,
 596                             uint64_t value)
 597 {
 598     /* Invalidate by IPA. This has to invalidate any structures that
 599      * contain only stage 2 translation information, but does not need
 600      * to apply to structures that contain combined stage 1 and stage 2
 601      * translation information.
 602      * This must NOP if EL2 isn't implemented or SCR_EL3.NS is zero.
 603      */
 604     CPUState *cs = ENV_GET_CPU(env);
 605     uint64_t pageaddr;
 606
 607     if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
 608         return;
 609     }
 610
 611     pageaddr = sextract64(value << 12, 0, 40);
 612
 613     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S2NS, -1);
 614 }
 615
 616 static void tlbiipas2_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 617                                uint64_t value)
 618 {
 619     CPUState *other_cs;
 620     uint64_t pageaddr;
 621
 622     if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
 623         return;
 624     }
 625
 626     pageaddr = sextract64(value << 12, 0, 40);
 627
 628     CPU_FOREACH(other_cs) {
 629         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S2NS, -1);
 630     }
 631 }
 632
 633 static void tlbiall_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
 634                               uint64_t value)
 635 {
 636     CPUState *cs = ENV_GET_CPU(env);
 637
 638     tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E2, -1);
 639 }
 640
 641 static void tlbiall_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 642                                  uint64_t value)
 643 {
 644     CPUState *other_cs;
 645
 646     CPU_FOREACH(other_cs) {
 647         tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E2, -1);
 648     }
 649 }
 650
 651 static void tlbimva_hyp_write(CPUARMState *env, const ARMCPRegInfo *ri,
 652                               uint64_t value)
 653 {
 654     CPUState *cs = ENV_GET_CPU(env);
 655     uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
 656
 657     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E2, -1);
 658 }
 659
 660 static void tlbimva_hyp_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 661                                  uint64_t value)
 662 {
 663     CPUState *other_cs;
 664     uint64_t pageaddr = value & ~MAKE_64BIT_MASK(0, 12);
 665
 666     CPU_FOREACH(other_cs) {
 667         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E2, -1);
 668     }
 669 }
 670
 671 static const ARMCPRegInfo cp_reginfo[] = {
 672     /* Define the secure and non-secure FCSE identifier CP registers
 673      * separately because there is no secure bank in V8 (no _EL3).  This allows
 674      * the secure register to be properly reset and migrated. There is also no
 675      * v8 EL1 version of the register so the non-secure instance stands alone.
 676      */
 677     { .name = "FCSEIDR(NS)",
 678       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 679       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 680       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
 681       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 682     { .name = "FCSEIDR(S)",
 683       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 684       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 685       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
 686       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 687     /* Define the secure and non-secure context identifier CP registers
 688      * separately because there is no secure bank in V8 (no _EL3).  This allows
 689      * the secure register to be properly reset and migrated.  In the
 690      * non-secure case, the 32-bit register will have reset and migration
 691      * disabled during registration as it is handled by the 64-bit instance.
 692      */
 693     { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
 694       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 695       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 696       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
 697       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 698     { .name = "CONTEXTIDR(S)", .state = ARM_CP_STATE_AA32,
 699       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 700       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 701       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
 702       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 703     REGINFO_SENTINEL
 704 };
 705
 706 static const ARMCPRegInfo not_v8_cp_reginfo[] = {
 707     /* NB: Some of these registers exist in v8 but with more precise
 708      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
 709      */
 710     /* MMU Domain access control / MPU write buffer control */
 711     { .name = "DACR",
 712       .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
 713       .access = PL1_RW, .resetvalue = 0,
 714       .writefn = dacr_write, .raw_writefn = raw_write,
 715       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
 716                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
 717     /* ARMv7 allocates a range of implementation defined TLB LOCKDOWN regs.
 718      * For v6 and v5, these mappings are overly broad.
 719      */
 720     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 0,
 721       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 722     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 1,
 723       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 724     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 4,
 725       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 726     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = 8,
 727       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 728     /* Cache maintenance ops; some of this space may be overridden later. */
 729     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
 730       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
 731       .type = ARM_CP_NOP | ARM_CP_OVERRIDE },
 732     REGINFO_SENTINEL
 733 };
 734
 735 static const ARMCPRegInfo not_v6_cp_reginfo[] = {
 736     /* Not all pre-v6 cores implemented this WFI, so this is slightly
 737      * over-broad.
 738      */
 739     { .name = "WFI_v5", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = 2,
 740       .access = PL1_W, .type = ARM_CP_WFI },
 741     REGINFO_SENTINEL
 742 };
 743
 744 static const ARMCPRegInfo not_v7_cp_reginfo[] = {
 745     /* Standard v6 WFI (also used in some pre-v6 cores); not in v7 (which
 746      * is UNPREDICTABLE; we choose to NOP as most implementations do).
 747      */
 748     { .name = "WFI_v6", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 749       .access = PL1_W, .type = ARM_CP_WFI },
 750     /* L1 cache lockdown. Not architectural in v6 and earlier but in practice
 751      * implemented in 926, 946, 1026, 1136, 1176 and 11MPCore. StrongARM and
 752      * OMAPCP will override this space.
 753      */
 754     { .name = "DLOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 0,
 755       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_data),
 756       .resetvalue = 0 },
 757     { .name = "ILOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 1,
 758       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_insn),
 759       .resetvalue = 0 },
 760     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
 761     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
 762       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
 763       .resetvalue = 0 },
 764     /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
 765      * implementing it as RAZ means the "debug architecture version" bits
 766      * will read as a reserved value, which should cause Linux to not try
 767      * to use the debug hardware.
 768      */
 769     { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
 770       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
 771     /* MMU TLB control. Note that the wildcarding means we cover not just
 772      * the unified TLB ops but also the dside/iside/inner-shareable variants.
 773      */
 774     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
 775       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
 776       .type = ARM_CP_NO_RAW },
 777     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
 778       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
 779       .type = ARM_CP_NO_RAW },
 780     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
 781       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
 782       .type = ARM_CP_NO_RAW },
 783     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
 784       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
 785       .type = ARM_CP_NO_RAW },
 786     { .name = "PRRR", .cp = 15, .crn = 10, .crm = 2,
 787       .opc1 = 0, .opc2 = 0, .access = PL1_RW, .type = ARM_CP_NOP },
 788     { .name = "NMRR", .cp = 15, .crn = 10, .crm = 2,
 789       .opc1 = 0, .opc2 = 1, .access = PL1_RW, .type = ARM_CP_NOP },
 790     REGINFO_SENTINEL
 791 };
 792
 793 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 794                         uint64_t value)
 795 {
 796     uint32_t mask = 0;
 797
 798     /* In ARMv8 most bits of CPACR_EL1 are RES0. */
 799     if (!arm_feature(env, ARM_FEATURE_V8)) {
 800         /* ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
 801          * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
 802          * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
 803          */
 804         if (arm_feature(env, ARM_FEATURE_VFP)) {
 805             /* VFP coprocessor: cp10 & cp11 [23:20] */
 806             mask |= (1 << 31) | (1 << 30) | (0xf << 20);
 807
 808             if (!arm_feature(env, ARM_FEATURE_NEON)) {
 809                 /* ASEDIS [31] bit is RAO/WI */
 810                 value |= (1 << 31);
 811             }
 812
 813             /* VFPv3 and upwards with NEON implement 32 double precision
 814              * registers (D0-D31).
 815              */
 816             if (!arm_feature(env, ARM_FEATURE_NEON) ||
 817                     !arm_feature(env, ARM_FEATURE_VFP3)) {
 818                 /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
 819                 value |= (1 << 30);
 820             }
 821         }
 822         value &= mask;
 823     }
 824     env->cp15.cpacr_el1 = value;
 825 }
 826
 827 static CPAccessResult cpacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
 828                                    bool isread)
 829 {
 830     if (arm_feature(env, ARM_FEATURE_V8)) {
 831         /* Check if CPACR accesses are to be trapped to EL2 */
 832         if (arm_current_el(env) == 1 &&
 833             (env->cp15.cptr_el[2] & CPTR_TCPAC) && !arm_is_secure(env)) {
 834             return CP_ACCESS_TRAP_EL2;
 835         /* Check if CPACR accesses are to be trapped to EL3 */
 836         } else if (arm_current_el(env) < 3 &&
 837                    (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
 838             return CP_ACCESS_TRAP_EL3;
 839         }
 840     }
 841
 842     return CP_ACCESS_OK;
 843 }
 844
 845 static CPAccessResult cptr_access(CPUARMState *env, const ARMCPRegInfo *ri,
 846                                   bool isread)
 847 {
 848     /* Check if CPTR accesses are set to trap to EL3 */
 849     if (arm_current_el(env) == 2 && (env->cp15.cptr_el[3] & CPTR_TCPAC)) {
 850         return CP_ACCESS_TRAP_EL3;
 851     }
 852
 853     return CP_ACCESS_OK;
 854 }
 855
 856 static const ARMCPRegInfo v6_cp_reginfo[] = {
 857     /* prefetch by MVA in v6, NOP in v7 */
 858     { .name = "MVA_prefetch",
 859       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
 860       .access = PL1_W, .type = ARM_CP_NOP },
 861     /* We need to break the TB after ISB to execute self-modifying code
 862      * correctly and also to take any pending interrupts immediately.
 863      * So use arm_cp_write_ignore() function instead of ARM_CP_NOP flag.
 864      */
 865     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
 866       .access = PL0_W, .type = ARM_CP_NO_RAW, .writefn = arm_cp_write_ignore },
 867     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
 868       .access = PL0_W, .type = ARM_CP_NOP },
 869     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
 870       .access = PL0_W, .type = ARM_CP_NOP },
 871     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
 872       .access = PL1_RW,
 873       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
 874                              offsetof(CPUARMState, cp15.ifar_ns) },
 875       .resetvalue = 0, },
 876     /* Watchpoint Fault Address Register : should actually only be present
 877      * for 1136, 1176, 11MPCore.
 878      */
 879     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
 880       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
 881     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
 882       .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2, .accessfn = cpacr_access,
 883       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.cpacr_el1),
 884       .resetvalue = 0, .writefn = cpacr_write },
 885     REGINFO_SENTINEL
 886 };
 887
 888 static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri,
 889                                    bool isread)
 890 {
 891     /* Performance monitor registers user accessibility is controlled
 892      * by PMUSERENR. MDCR_EL2.TPM and MDCR_EL3.TPM allow configurable
 893      * trapping to EL2 or EL3 for other accesses.
 894      */
 895     int el = arm_current_el(env);
 896
 897     if (el == 0 && !env->cp15.c9_pmuserenr) {
 898         return CP_ACCESS_TRAP;
 899     }
 900     if (el < 2 && (env->cp15.mdcr_el2 & MDCR_TPM)
 901         && !arm_is_secure_below_el3(env)) {
 902         return CP_ACCESS_TRAP_EL2;
 903     }
 904     if (el < 3 && (env->cp15.mdcr_el3 & MDCR_TPM)) {
 905         return CP_ACCESS_TRAP_EL3;
 906     }
 907
 908     return CP_ACCESS_OK;
 909 }
 910
 911 #ifndef CONFIG_USER_ONLY
 912
 913 static inline bool arm_ccnt_enabled(CPUARMState *env)
 914 {
 915     /* This does not support checking PMCCFILTR_EL0 register */
 916
 917     if (!(env->cp15.c9_pmcr & PMCRE)) {
 918         return false;
 919     }
 920
 921     return true;
 922 }
 923
 924 void pmccntr_sync(CPUARMState *env)
 925 {
 926     uint64_t temp_ticks;
 927
 928     temp_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
 929                           ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 930
 931     if (env->cp15.c9_pmcr & PMCRD) {
 932         /* Increment once every 64 processor clock cycles */
 933         temp_ticks /= 64;
 934     }
 935
 936     if (arm_ccnt_enabled(env)) {
 937         env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
 938     }
 939 }
 940
 941 static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 942                        uint64_t value)
 943 {
 944     pmccntr_sync(env);
 945
 946     if (value & PMCRC) {
 947         /* The counter has been reset */
 948         env->cp15.c15_ccnt = 0;
 949     }
 950
 951     /* only the DP, X, D and E bits are writable */
 952     env->cp15.c9_pmcr &= ~0x39;
 953     env->cp15.c9_pmcr |= (value & 0x39);
 954
 955     pmccntr_sync(env);
 956 }
 957
 958 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 959 {
 960     uint64_t total_ticks;
 961
 962     if (!arm_ccnt_enabled(env)) {
 963         /* Counter is disabled, do not change value */
 964         return env->cp15.c15_ccnt;
 965     }
 966
 967     total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
 968                            ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 969
 970     if (env->cp15.c9_pmcr & PMCRD) {
 971         /* Increment once every 64 processor clock cycles */
 972         total_ticks /= 64;
 973     }
 974     return total_ticks - env->cp15.c15_ccnt;
 975 }
 976
 977 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 978                         uint64_t value)
 979 {
 980     uint64_t total_ticks;
 981
 982     if (!arm_ccnt_enabled(env)) {
 983         /* Counter is disabled, set the absolute value */
 984         env->cp15.c15_ccnt = value;
 985         return;
 986     }
 987
 988     total_ticks = muldiv64(qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL),
 989                            ARM_CPU_FREQ, NANOSECONDS_PER_SECOND);
 990
 991     if (env->cp15.c9_pmcr & PMCRD) {
 992         /* Increment once every 64 processor clock cycles */
 993         total_ticks /= 64;
 994     }
 995     env->cp15.c15_ccnt = total_ticks - value;
 996 }
 997
 998 static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
 999                             uint64_t value)
1000 {
1001     uint64_t cur_val = pmccntr_read(env, NULL);
1002
1003     pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
1004 }
1005
1006 #else /* CONFIG_USER_ONLY */
1007
1008 void pmccntr_sync(CPUARMState *env)
1009 {
1010 }
1011
1012 #endif
1013
1014 static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1015                             uint64_t value)
1016 {
1017     pmccntr_sync(env);
1018     env->cp15.pmccfiltr_el0 = value & 0x7E000000;
1019     pmccntr_sync(env);
1020 }
1021
1022 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1023                             uint64_t value)
1024 {
1025     value &= (1 << 31);
1026     env->cp15.c9_pmcnten |= value;
1027 }
1028
1029 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1030                              uint64_t value)
1031 {
1032     value &= (1 << 31);
1033     env->cp15.c9_pmcnten &= ~value;
1034 }
1035
1036 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1037                          uint64_t value)
1038 {
1039     env->cp15.c9_pmovsr &= ~value;
1040 }
1041
1042 static void pmxevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
1043                              uint64_t value)
1044 {
1045     env->cp15.c9_pmxevtyper = value & 0xff;
1046 }
1047
1048 static void pmuserenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1049                             uint64_t value)
1050 {
1051     env->cp15.c9_pmuserenr = value & 1;
1052 }
1053
1054 static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
1055                              uint64_t value)
1056 {
1057     /* We have no event counters so only the C bit can be changed */
1058     value &= (1 << 31);
1059     env->cp15.c9_pminten |= value;
1060 }
1061
1062 static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1063                              uint64_t value)
1064 {
1065     value &= (1 << 31);
1066     env->cp15.c9_pminten &= ~value;
1067 }
1068
1069 static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
1070                        uint64_t value)
1071 {
1072     /* Note that even though the AArch64 view of this register has bits
1073      * [10:0] all RES0 we can only mask the bottom 5, to comply with the
1074      * architectural requirements for bits which are RES0 only in some
1075      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
1076      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
1077      */
1078     raw_write(env, ri, value & ~0x1FULL);
1079 }
1080
1081 static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1082 {
1083     /* We only mask off bits that are RES0 both for AArch64 and AArch32.
1084      * For bits that vary between AArch32/64, code needs to check the
1085      * current execution mode before directly using the feature bit.
1086      */
1087     uint32_t valid_mask = SCR_AARCH64_MASK | SCR_AARCH32_MASK;
1088
1089     if (!arm_feature(env, ARM_FEATURE_EL2)) {
1090         valid_mask &= ~SCR_HCE;
1091
1092         /* On ARMv7, SMD (or SCD as it is called in v7) is only
1093          * supported if EL2 exists. The bit is UNK/SBZP when
1094          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
1095          * when EL2 is unavailable.
1096          * On ARMv8, this bit is always available.
1097          */
1098         if (arm_feature(env, ARM_FEATURE_V7) &&
1099             !arm_feature(env, ARM_FEATURE_V8)) {
1100             valid_mask &= ~SCR_SMD;
1101         }
1102     }
1103
1104     /* Clear all-context RES0 bits.  */
1105     value &= valid_mask;
1106     raw_write(env, ri, value);
1107 }
1108
1109 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1110 {
1111     ARMCPU *cpu = arm_env_get_cpu(env);
1112
1113     /* Acquire the CSSELR index from the bank corresponding to the CCSIDR
1114      * bank
1115      */
1116     uint32_t index = A32_BANKED_REG_GET(env, csselr,
1117                                         ri->secure & ARM_CP_SECSTATE_S);
1118
1119     return cpu->ccsidr[index];
1120 }
1121
1122 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1123                          uint64_t value)
1124 {
1125     raw_write(env, ri, value & 0xf);
1126 }
1127
1128 static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1129 {
1130     CPUState *cs = ENV_GET_CPU(env);
1131     uint64_t ret = 0;
1132
1133     if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
1134         ret |= CPSR_I;
1135     }
1136     if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
1137         ret |= CPSR_F;
1138     }
1139     /* External aborts are not possible in QEMU so A bit is always clear */
1140     return ret;
1141 }
1142
1143 static const ARMCPRegInfo v7_cp_reginfo[] = {
1144     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
1145     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
1146       .access = PL1_W, .type = ARM_CP_NOP },
1147     /* Performance monitors are implementation defined in v7,
1148      * but with an ARM recommended set of registers, which we
1149      * follow (although we don't actually implement any counters)
1150      *
1151      * Performance registers fall into three categories:
1152      *  (a) always UNDEF in PL0, RW in PL1 (PMINTENSET, PMINTENCLR)
1153      *  (b) RO in PL0 (ie UNDEF on write), RW in PL1 (PMUSERENR)
1154      *  (c) UNDEF in PL0 if PMUSERENR.EN==0, otherwise accessible (all others)
1155      * For the cases controlled by PMUSERENR we must set .access to PL0_RW
1156      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
1157      */
1158     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
1159       .access = PL0_RW, .type = ARM_CP_ALIAS,
1160       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
1161       .writefn = pmcntenset_write,
1162       .accessfn = pmreg_access,
1163       .raw_writefn = raw_write },
1164     { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64,
1165       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
1166       .access = PL0_RW, .accessfn = pmreg_access,
1167       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
1168       .writefn = pmcntenset_write, .raw_writefn = raw_write },
1169     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
1170       .access = PL0_RW,
1171       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
1172       .accessfn = pmreg_access,
1173       .writefn = pmcntenclr_write,
1174       .type = ARM_CP_ALIAS },
1175     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
1176       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
1177       .access = PL0_RW, .accessfn = pmreg_access,
1178       .type = ARM_CP_ALIAS,
1179       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
1180       .writefn = pmcntenclr_write },
1181     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
1182       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
1183       .accessfn = pmreg_access,
1184       .writefn = pmovsr_write,
1185       .raw_writefn = raw_write },
1186     { .name = "PMOVSCLR_EL0", .state = ARM_CP_STATE_AA64,
1187       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 3,
1188       .access = PL0_RW, .accessfn = pmreg_access,
1189       .type = ARM_CP_ALIAS,
1190       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
1191       .writefn = pmovsr_write,
1192       .raw_writefn = raw_write },
1193     /* Unimplemented so WI. */
1194     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
1195       .access = PL0_W, .accessfn = pmreg_access, .type = ARM_CP_NOP },
1196     /* Since we don't implement any events, writing to PMSELR is UNPREDICTABLE.
1197      * We choose to RAZ/WI.
1198      */
1199     { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5,
1200       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
1201       .accessfn = pmreg_access },
1202 #ifndef CONFIG_USER_ONLY
1203     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
1204       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_IO,
1205       .readfn = pmccntr_read, .writefn = pmccntr_write32,
1206       .accessfn = pmreg_access },
1207     { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
1208       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
1209       .access = PL0_RW, .accessfn = pmreg_access,
1210       .type = ARM_CP_IO,
1211       .readfn = pmccntr_read, .writefn = pmccntr_write, },
1212 #endif
1213     { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
1214       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
1215       .writefn = pmccfiltr_write,
1216       .access = PL0_RW, .accessfn = pmreg_access,
1217       .type = ARM_CP_IO,
1218       .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
1219       .resetvalue = 0, },
1220     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
1221       .access = PL0_RW,
1222       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmxevtyper),
1223       .accessfn = pmreg_access, .writefn = pmxevtyper_write,
1224       .raw_writefn = raw_write },
1225     /* Unimplemented, RAZ/WI. */
1226     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
1227       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
1228       .accessfn = pmreg_access },
1229     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
1230       .access = PL0_R | PL1_RW, .accessfn = access_tpm,
1231       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
1232       .resetvalue = 0,
1233       .writefn = pmuserenr_write, .raw_writefn = raw_write },
1234     { .name = "PMUSERENR_EL0", .state = ARM_CP_STATE_AA64,
1235       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 14, .opc2 = 0,
1236       .access = PL0_R | PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
1237       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
1238       .resetvalue = 0,
1239       .writefn = pmuserenr_write, .raw_writefn = raw_write },
1240     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
1241       .access = PL1_RW, .accessfn = access_tpm,
1242       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
1243       .resetvalue = 0,
1244       .writefn = pmintenset_write, .raw_writefn = raw_write },
1245     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
1246       .access = PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
1247       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
1248       .writefn = pmintenclr_write, },
1249     { .name = "PMINTENCLR_EL1", .state = ARM_CP_STATE_AA64,
1250       .opc0 = 3, .opc1 = 0, .crn = 9, .crm = 14, .opc2 = 2,
1251       .access = PL1_RW, .accessfn = access_tpm, .type = ARM_CP_ALIAS,
1252       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
1253       .writefn = pmintenclr_write },
1254     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
1255       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
1256       .access = PL1_RW, .writefn = vbar_write,
1257       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
1258                              offsetof(CPUARMState, cp15.vbar_ns) },
1259       .resetvalue = 0 },
1260     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
1261       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
1262       .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_RAW },
1263     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
1264       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
1265       .access = PL1_RW, .writefn = csselr_write, .resetvalue = 0,
1266       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
1267                              offsetof(CPUARMState, cp15.csselr_ns) } },
1268     /* Auxiliary ID register: this actually has an IMPDEF value but for now
1269      * just RAZ for all cores:
1270      */
1271     { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
1272       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
1273       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
1274     /* Auxiliary fault status registers: these also are IMPDEF, and we
1275      * choose to RAZ/WI for all cores.
1276      */
1277     { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
1278       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
1279       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
1280     { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
1281       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
1282       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
1283     /* MAIR can just read-as-written because we don't implement caches
1284      * and so don't need to care about memory attributes.
1285      */
1286     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
1287       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
1288       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
1289       .resetvalue = 0 },
1290     { .name = "MAIR_EL3", .state = ARM_CP_STATE_AA64,
1291       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 2, .opc2 = 0,
1292       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[3]),
1293       .resetvalue = 0 },
1294     /* For non-long-descriptor page tables these are PRRR and NMRR;
1295      * regardless they still act as reads-as-written for QEMU.
1296      */
1297      /* MAIR0/1 are defined separately from their 64-bit counterpart which
1298       * allows them to assign the correct fieldoffset based on the endianness
1299       * handled in the field definitions.
1300       */
1301     { .name = "MAIR0", .state = ARM_CP_STATE_AA32,
1302       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0, .access = PL1_RW,
1303       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
1304                              offsetof(CPUARMState, cp15.mair0_ns) },
1305       .resetfn = arm_cp_reset_ignore },
1306     { .name = "MAIR1", .state = ARM_CP_STATE_AA32,
1307       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1, .access = PL1_RW,
1308       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
1309                              offsetof(CPUARMState, cp15.mair1_ns) },
1310       .resetfn = arm_cp_reset_ignore },
1311     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
1312       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
1313       .type = ARM_CP_NO_RAW, .access = PL1_R, .readfn = isr_read },
1314     /* 32 bit ITLB invalidates */
1315     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
1316       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1317     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
1318       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1319     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
1320       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1321     /* 32 bit DTLB invalidates */
1322     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
1323       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1324     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
1325       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1326     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
1327       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1328     /* 32 bit TLB invalidates */
1329     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
1330       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_write },
1331     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
1332       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
1333     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
1334       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiasid_write },
1335     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
1336       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
1337     REGINFO_SENTINEL
1338 };
1339
1340 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
1341     /* 32 bit TLB invalidates, Inner Shareable */
1342     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
1343       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbiall_is_write },
1344     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
1345       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
1346     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
1347       .type = ARM_CP_NO_RAW, .access = PL1_W,
1348       .writefn = tlbiasid_is_write },
1349     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
1350       .type = ARM_CP_NO_RAW, .access = PL1_W,
1351       .writefn = tlbimvaa_is_write },
1352     REGINFO_SENTINEL
1353 };
1354
1355 static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1356                         uint64_t value)
1357 {
1358     value &= 1;
1359     env->teecr = value;
1360 }
1361
1362 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri,
1363                                     bool isread)
1364 {
1365     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
1366         return CP_ACCESS_TRAP;
1367     }
1368     return CP_ACCESS_OK;
1369 }
1370
1371 static const ARMCPRegInfo t2ee_cp_reginfo[] = {
1372     { .name = "TEECR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 6, .opc2 = 0,
1373       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, teecr),
1374       .resetvalue = 0,
1375       .writefn = teecr_write },
1376     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
1377       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
1378       .accessfn = teehbr_access, .resetvalue = 0 },
1379     REGINFO_SENTINEL
1380 };
1381
1382 static const ARMCPRegInfo v6k_cp_reginfo[] = {
1383     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
1384       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
1385       .access = PL0_RW,
1386       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
1387     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
1388       .access = PL0_RW,
1389       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
1390                              offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
1391       .resetfn = arm_cp_reset_ignore },
1392     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
1393       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
1394       .access = PL0_R|PL1_W,
1395       .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
1396       .resetvalue = 0},
1397     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
1398       .access = PL0_R|PL1_W,
1399       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
1400                              offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
1401       .resetfn = arm_cp_reset_ignore },
1402     { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
1403       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
1404       .access = PL1_RW,
1405       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
1406     { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
1407       .access = PL1_RW,
1408       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
1409                              offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
1410       .resetvalue = 0 },
1411     REGINFO_SENTINEL
1412 };
1413
1414 #ifndef CONFIG_USER_ONLY
1415
1416 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri,
1417                                        bool isread)
1418 {
1419     /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero.
1420      * Writable only at the highest implemented exception level.
1421      */
1422     int el = arm_current_el(env);
1423
1424     switch (el) {
1425     case 0:
1426         if (!extract32(env->cp15.c14_cntkctl, 0, 2)) {
1427             return CP_ACCESS_TRAP;
1428         }
1429         break;
1430     case 1:
1431         if (!isread && ri->state == ARM_CP_STATE_AA32 &&
1432             arm_is_secure_below_el3(env)) {
1433             /* Accesses from 32-bit Secure EL1 UNDEF (*not* trap to EL3!) */
1434             return CP_ACCESS_TRAP_UNCATEGORIZED;
1435         }
1436         break;
1437     case 2:
1438     case 3:
1439         break;
1440     }
1441
1442     if (!isread && el < arm_highest_el(env)) {
1443         return CP_ACCESS_TRAP_UNCATEGORIZED;
1444     }
1445
1446     return CP_ACCESS_OK;
1447 }
1448
1449 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx,
1450                                         bool isread)
1451 {
1452     unsigned int cur_el = arm_current_el(env);
1453     bool secure = arm_is_secure(env);
1454
1455     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
1456     if (cur_el == 0 &&
1457         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
1458         return CP_ACCESS_TRAP;
1459     }
1460
1461     if (arm_feature(env, ARM_FEATURE_EL2) &&
1462         timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
1463         !extract32(env->cp15.cnthctl_el2, 0, 1)) {
1464         return CP_ACCESS_TRAP_EL2;
1465     }
1466     return CP_ACCESS_OK;
1467 }
1468
1469 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx,
1470                                       bool isread)
1471 {
1472     unsigned int cur_el = arm_current_el(env);
1473     bool secure = arm_is_secure(env);
1474
1475     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
1476      * EL0[PV]TEN is zero.
1477      */
1478     if (cur_el == 0 &&
1479         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
1480         return CP_ACCESS_TRAP;
1481     }
1482
1483     if (arm_feature(env, ARM_FEATURE_EL2) &&
1484         timeridx == GTIMER_PHYS && !secure && cur_el < 2 &&
1485         !extract32(env->cp15.cnthctl_el2, 1, 1)) {
1486         return CP_ACCESS_TRAP_EL2;
1487     }
1488     return CP_ACCESS_OK;
1489 }
1490
1491 static CPAccessResult gt_pct_access(CPUARMState *env,
1492                                     const ARMCPRegInfo *ri,
1493                                     bool isread)
1494 {
1495     return gt_counter_access(env, GTIMER_PHYS, isread);
1496 }
1497
1498 static CPAccessResult gt_vct_access(CPUARMState *env,
1499                                     const ARMCPRegInfo *ri,
1500                                     bool isread)
1501 {
1502     return gt_counter_access(env, GTIMER_VIRT, isread);
1503 }
1504
1505 static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
1506                                        bool isread)
1507 {
1508     return gt_timer_access(env, GTIMER_PHYS, isread);
1509 }
1510
1511 static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri,
1512                                        bool isread)
1513 {
1514     return gt_timer_access(env, GTIMER_VIRT, isread);
1515 }
1516
1517 static CPAccessResult gt_stimer_access(CPUARMState *env,
1518                                        const ARMCPRegInfo *ri,
1519                                        bool isread)
1520 {
1521     /* The AArch64 register view of the secure physical timer is
1522      * always accessible from EL3, and configurably accessible from
1523      * Secure EL1.
1524      */
1525     switch (arm_current_el(env)) {
1526     case 1:
1527         if (!arm_is_secure(env)) {
1528             return CP_ACCESS_TRAP;
1529         }
1530         if (!(env->cp15.scr_el3 & SCR_ST)) {
1531             return CP_ACCESS_TRAP_EL3;
1532         }
1533         return CP_ACCESS_OK;
1534     case 0:
1535     case 2:
1536         return CP_ACCESS_TRAP;
1537     case 3:
1538         return CP_ACCESS_OK;
1539     default:
1540         g_assert_not_reached();
1541     }
1542 }
1543
1544 static uint64_t gt_get_countervalue(CPUARMState *env)
1545 {
1546     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / GTIMER_SCALE;
1547 }
1548
1549 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
1550 {
1551     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
1552
1553     if (gt->ctl & 1) {
1554         /* Timer enabled: calculate and set current ISTATUS, irq, and
1555          * reset timer to when ISTATUS next has to change
1556          */
1557         uint64_t offset = timeridx == GTIMER_VIRT ?
1558                                       cpu->env.cp15.cntvoff_el2 : 0;
1559         uint64_t count = gt_get_countervalue(&cpu->env);
1560         /* Note that this must be unsigned 64 bit arithmetic: */
1561         int istatus = count - offset >= gt->cval;
1562         uint64_t nexttick;
1563
1564         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
1565         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1566                      (istatus && !(gt->ctl & 2)));
1567         if (istatus) {
1568             /* Next transition is when count rolls back over to zero */
1569             nexttick = UINT64_MAX;
1570         } else {
1571             /* Next transition is when we hit cval */
1572             nexttick = gt->cval + offset;
1573         }
1574         /* Note that the desired next expiry time might be beyond the
1575          * signed-64-bit range of a QEMUTimer -- in this case we just
1576          * set the timer for as far in the future as possible. When the
1577          * timer expires we will reset the timer for any remaining period.
1578          */
1579         if (nexttick > INT64_MAX / GTIMER_SCALE) {
1580             nexttick = INT64_MAX / GTIMER_SCALE;
1581         }
1582         timer_mod(cpu->gt_timer[timeridx], nexttick);
1583     } else {
1584         /* Timer disabled: ISTATUS and timer output always clear */
1585         gt->ctl &= ~4;
1586         qemu_set_irq(cpu->gt_timer_outputs[timeridx], 0);
1587         timer_del(cpu->gt_timer[timeridx]);
1588     }
1589 }
1590
1591 static void gt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri,
1592                            int timeridx)
1593 {
1594     ARMCPU *cpu = arm_env_get_cpu(env);
1595
1596     timer_del(cpu->gt_timer[timeridx]);
1597 }
1598
1599 static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
1600 {
1601     return gt_get_countervalue(env);
1602 }
1603
1604 static uint64_t gt_virt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
1605 {
1606     return gt_get_countervalue(env) - env->cp15.cntvoff_el2;
1607 }
1608
1609 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1610                           int timeridx,
1611                           uint64_t value)
1612 {
1613     env->cp15.c14_timer[timeridx].cval = value;
1614     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1615 }
1616
1617 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri,
1618                              int timeridx)
1619 {
1620     uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
1621
1622     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
1623                       (gt_get_countervalue(env) - offset));
1624 }
1625
1626 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1627                           int timeridx,
1628                           uint64_t value)
1629 {
1630     uint64_t offset = timeridx == GTIMER_VIRT ? env->cp15.cntvoff_el2 : 0;
1631
1632     env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) - offset +
1633                                          sextract64(value, 0, 32);
1634     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1635 }
1636
1637 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1638                          int timeridx,
1639                          uint64_t value)
1640 {
1641     ARMCPU *cpu = arm_env_get_cpu(env);
1642     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
1643
1644     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
1645     if ((oldval ^ value) & 1) {
1646         /* Enable toggled */
1647         gt_recalc_timer(cpu, timeridx);
1648     } else if ((oldval ^ value) & 2) {
1649         /* IMASK toggled: don't need to recalculate,
1650          * just set the interrupt line based on ISTATUS
1651          */
1652         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1653                      (oldval & 4) && !(value & 2));
1654     }
1655 }
1656
1657 static void gt_phys_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1658 {
1659     gt_timer_reset(env, ri, GTIMER_PHYS);
1660 }
1661
1662 static void gt_phys_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1663                                uint64_t value)
1664 {
1665     gt_cval_write(env, ri, GTIMER_PHYS, value);
1666 }
1667
1668 static uint64_t gt_phys_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1669 {
1670     return gt_tval_read(env, ri, GTIMER_PHYS);
1671 }
1672
1673 static void gt_phys_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1674                                uint64_t value)
1675 {
1676     gt_tval_write(env, ri, GTIMER_PHYS, value);
1677 }
1678
1679 static void gt_phys_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1680                               uint64_t value)
1681 {
1682     gt_ctl_write(env, ri, GTIMER_PHYS, value);
1683 }
1684
1685 static void gt_virt_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1686 {
1687     gt_timer_reset(env, ri, GTIMER_VIRT);
1688 }
1689
1690 static void gt_virt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1691                                uint64_t value)
1692 {
1693     gt_cval_write(env, ri, GTIMER_VIRT, value);
1694 }
1695
1696 static uint64_t gt_virt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1697 {
1698     return gt_tval_read(env, ri, GTIMER_VIRT);
1699 }
1700
1701 static void gt_virt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1702                                uint64_t value)
1703 {
1704     gt_tval_write(env, ri, GTIMER_VIRT, value);
1705 }
1706
1707 static void gt_virt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1708                               uint64_t value)
1709 {
1710     gt_ctl_write(env, ri, GTIMER_VIRT, value);
1711 }
1712
1713 static void gt_cntvoff_write(CPUARMState *env, const ARMCPRegInfo *ri,
1714                               uint64_t value)
1715 {
1716     ARMCPU *cpu = arm_env_get_cpu(env);
1717
1718     raw_write(env, ri, value);
1719     gt_recalc_timer(cpu, GTIMER_VIRT);
1720 }
1721
1722 static void gt_hyp_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1723 {
1724     gt_timer_reset(env, ri, GTIMER_HYP);
1725 }
1726
1727 static void gt_hyp_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1728                               uint64_t value)
1729 {
1730     gt_cval_write(env, ri, GTIMER_HYP, value);
1731 }
1732
1733 static uint64_t gt_hyp_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1734 {
1735     return gt_tval_read(env, ri, GTIMER_HYP);
1736 }
1737
1738 static void gt_hyp_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1739                               uint64_t value)
1740 {
1741     gt_tval_write(env, ri, GTIMER_HYP, value);
1742 }
1743
1744 static void gt_hyp_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1745                               uint64_t value)
1746 {
1747     gt_ctl_write(env, ri, GTIMER_HYP, value);
1748 }
1749
1750 static void gt_sec_timer_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1751 {
1752     gt_timer_reset(env, ri, GTIMER_SEC);
1753 }
1754
1755 static void gt_sec_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1756                               uint64_t value)
1757 {
1758     gt_cval_write(env, ri, GTIMER_SEC, value);
1759 }
1760
1761 static uint64_t gt_sec_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1762 {
1763     return gt_tval_read(env, ri, GTIMER_SEC);
1764 }
1765
1766 static void gt_sec_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1767                               uint64_t value)
1768 {
1769     gt_tval_write(env, ri, GTIMER_SEC, value);
1770 }
1771
1772 static void gt_sec_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1773                               uint64_t value)
1774 {
1775     gt_ctl_write(env, ri, GTIMER_SEC, value);
1776 }
1777
1778 void arm_gt_ptimer_cb(void *opaque)
1779 {
1780     ARMCPU *cpu = opaque;
1781
1782     gt_recalc_timer(cpu, GTIMER_PHYS);
1783 }
1784
1785 void arm_gt_vtimer_cb(void *opaque)
1786 {
1787     ARMCPU *cpu = opaque;
1788
1789     gt_recalc_timer(cpu, GTIMER_VIRT);
1790 }
1791
1792 void arm_gt_htimer_cb(void *opaque)
1793 {
1794     ARMCPU *cpu = opaque;
1795
1796     gt_recalc_timer(cpu, GTIMER_HYP);
1797 }
1798
1799 void arm_gt_stimer_cb(void *opaque)
1800 {
1801     ARMCPU *cpu = opaque;
1802
1803     gt_recalc_timer(cpu, GTIMER_SEC);
1804 }
1805
1806 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1807     /* Note that CNTFRQ is purely reads-as-written for the benefit
1808      * of software; writing it doesn't actually change the timer frequency.
1809      * Our reset value matches the fixed frequency we implement the timer at.
1810      */
1811     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
1812       .type = ARM_CP_ALIAS,
1813       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1814       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
1815     },
1816     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
1817       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
1818       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1819       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
1820       .resetvalue = (1000 * 1000 * 1000) / GTIMER_SCALE,
1821     },
1822     /* overall control: mostly access permissions */
1823     { .name = "CNTKCTL", .state = ARM_CP_STATE_BOTH,
1824       .opc0 = 3, .opc1 = 0, .crn = 14, .crm = 1, .opc2 = 0,
1825       .access = PL1_RW,
1826       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntkctl),
1827       .resetvalue = 0,
1828     },
1829     /* per-timer control */
1830     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
1831       .secure = ARM_CP_SECSTATE_NS,
1832       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1833       .accessfn = gt_ptimer_access,
1834       .fieldoffset = offsetoflow32(CPUARMState,
1835                                    cp15.c14_timer[GTIMER_PHYS].ctl),
1836       .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
1837     },
1838     { .name = "CNTP_CTL(S)",
1839       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
1840       .secure = ARM_CP_SECSTATE_S,
1841       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1842       .accessfn = gt_ptimer_access,
1843       .fieldoffset = offsetoflow32(CPUARMState,
1844                                    cp15.c14_timer[GTIMER_SEC].ctl),
1845       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
1846     },
1847     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
1848       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
1849       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1850       .accessfn = gt_ptimer_access,
1851       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
1852       .resetvalue = 0,
1853       .writefn = gt_phys_ctl_write, .raw_writefn = raw_write,
1854     },
1855     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
1856       .type = ARM_CP_IO | ARM_CP_ALIAS, .access = PL1_RW | PL0_R,
1857       .accessfn = gt_vtimer_access,
1858       .fieldoffset = offsetoflow32(CPUARMState,
1859                                    cp15.c14_timer[GTIMER_VIRT].ctl),
1860       .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
1861     },
1862     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
1863       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
1864       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1865       .accessfn = gt_vtimer_access,
1866       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
1867       .resetvalue = 0,
1868       .writefn = gt_virt_ctl_write, .raw_writefn = raw_write,
1869     },
1870     /* TimerValue views: a 32 bit downcounting view of the underlying state */
1871     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
1872       .secure = ARM_CP_SECSTATE_NS,
1873       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1874       .accessfn = gt_ptimer_access,
1875       .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
1876     },
1877     { .name = "CNTP_TVAL(S)",
1878       .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
1879       .secure = ARM_CP_SECSTATE_S,
1880       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1881       .accessfn = gt_ptimer_access,
1882       .readfn = gt_sec_tval_read, .writefn = gt_sec_tval_write,
1883     },
1884     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1885       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
1886       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1887       .accessfn = gt_ptimer_access, .resetfn = gt_phys_timer_reset,
1888       .readfn = gt_phys_tval_read, .writefn = gt_phys_tval_write,
1889     },
1890     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
1891       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1892       .accessfn = gt_vtimer_access,
1893       .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
1894     },
1895     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1896       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
1897       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW | PL0_R,
1898       .accessfn = gt_vtimer_access, .resetfn = gt_virt_timer_reset,
1899       .readfn = gt_virt_tval_read, .writefn = gt_virt_tval_write,
1900     },
1901     /* The counter itself */
1902     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
1903       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1904       .accessfn = gt_pct_access,
1905       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1906     },
1907     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
1908       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
1909       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1910       .accessfn = gt_pct_access, .readfn = gt_cnt_read,
1911     },
1912     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
1913       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_RAW | ARM_CP_IO,
1914       .accessfn = gt_vct_access,
1915       .readfn = gt_virt_cnt_read, .resetfn = arm_cp_reset_ignore,
1916     },
1917     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
1918       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
1919       .access = PL0_R, .type = ARM_CP_NO_RAW | ARM_CP_IO,
1920       .accessfn = gt_vct_access, .readfn = gt_virt_cnt_read,
1921     },
1922     /* Comparison value, indicating when the timer goes off */
1923     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
1924       .secure = ARM_CP_SECSTATE_NS,
1925       .access = PL1_RW | PL0_R,
1926       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1927       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1928       .accessfn = gt_ptimer_access,
1929       .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
1930     },
1931     { .name = "CNTP_CVAL(S)", .cp = 15, .crm = 14, .opc1 = 2,
1932       .secure = ARM_CP_SECSTATE_S,
1933       .access = PL1_RW | PL0_R,
1934       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1935       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
1936       .accessfn = gt_ptimer_access,
1937       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
1938     },
1939     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1940       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
1941       .access = PL1_RW | PL0_R,
1942       .type = ARM_CP_IO,
1943       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1944       .resetvalue = 0, .accessfn = gt_ptimer_access,
1945       .writefn = gt_phys_cval_write, .raw_writefn = raw_write,
1946     },
1947     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
1948       .access = PL1_RW | PL0_R,
1949       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_ALIAS,
1950       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1951       .accessfn = gt_vtimer_access,
1952       .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
1953     },
1954     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1955       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
1956       .access = PL1_RW | PL0_R,
1957       .type = ARM_CP_IO,
1958       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1959       .resetvalue = 0, .accessfn = gt_vtimer_access,
1960       .writefn = gt_virt_cval_write, .raw_writefn = raw_write,
1961     },
1962     /* Secure timer -- this is actually restricted to only EL3
1963      * and configurably Secure-EL1 via the accessfn.
1964      */
1965     { .name = "CNTPS_TVAL_EL1", .state = ARM_CP_STATE_AA64,
1966       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 0,
1967       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL1_RW,
1968       .accessfn = gt_stimer_access,
1969       .readfn = gt_sec_tval_read,
1970       .writefn = gt_sec_tval_write,
1971       .resetfn = gt_sec_timer_reset,
1972     },
1973     { .name = "CNTPS_CTL_EL1", .state = ARM_CP_STATE_AA64,
1974       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 1,
1975       .type = ARM_CP_IO, .access = PL1_RW,
1976       .accessfn = gt_stimer_access,
1977       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].ctl),
1978       .resetvalue = 0,
1979       .writefn = gt_sec_ctl_write, .raw_writefn = raw_write,
1980     },
1981     { .name = "CNTPS_CVAL_EL1", .state = ARM_CP_STATE_AA64,
1982       .opc0 = 3, .opc1 = 7, .crn = 14, .crm = 2, .opc2 = 2,
1983       .type = ARM_CP_IO, .access = PL1_RW,
1984       .accessfn = gt_stimer_access,
1985       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_SEC].cval),
1986       .writefn = gt_sec_cval_write, .raw_writefn = raw_write,
1987     },
1988     REGINFO_SENTINEL
1989 };
1990
1991 #else
1992 /* In user-mode none of the generic timer registers are accessible,
1993  * and their implementation depends on QEMU_CLOCK_VIRTUAL and qdev gpio outputs,
1994  * so instead just don't register any of them.
1995  */
1996 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1997     REGINFO_SENTINEL
1998 };
1999
2000 #endif
2001
2002 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
2003 {
2004     if (arm_feature(env, ARM_FEATURE_LPAE)) {
2005         raw_write(env, ri, value);
2006     } else if (arm_feature(env, ARM_FEATURE_V7)) {
2007         raw_write(env, ri, value & 0xfffff6ff);
2008     } else {
2009         raw_write(env, ri, value & 0xfffff1ff);
2010     }
2011 }
2012
2013 #ifndef CONFIG_USER_ONLY
2014 /* get_phys_addr() isn't present for user-mode-only targets */
2015
2016 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri,
2017                                  bool isread)
2018 {
2019     if (ri->opc2 & 4) {
2020         /* The ATS12NSO* operations must trap to EL3 if executed in
2021          * Secure EL1 (which can only happen if EL3 is AArch64).
2022          * They are simply UNDEF if executed from NS EL1.
2023          * They function normally from EL2 or EL3.
2024          */
2025         if (arm_current_el(env) == 1) {
2026             if (arm_is_secure_below_el3(env)) {
2027                 return CP_ACCESS_TRAP_UNCATEGORIZED_EL3;
2028             }
2029             return CP_ACCESS_TRAP_UNCATEGORIZED;
2030         }
2031     }
2032     return CP_ACCESS_OK;
2033 }
2034
2035 static uint64_t do_ats_write(CPUARMState *env, uint64_t value,
2036                              int access_type, ARMMMUIdx mmu_idx)
2037 {
2038     hwaddr phys_addr;
2039     target_ulong page_size;
2040     int prot;
2041     uint32_t fsr;
2042     bool ret;
2043     uint64_t par64;
2044     MemTxAttrs attrs = {};
2045     ARMMMUFaultInfo fi = {};
2046
2047     ret = get_phys_addr(env, value, access_type, mmu_idx,
2048                         &phys_addr, &attrs, &prot, &page_size, &fsr, &fi);
2049     if (extended_addresses_enabled(env)) {
2050         /* fsr is a DFSR/IFSR value for the long descriptor
2051          * translation table format, but with WnR always clear.
2052          * Convert it to a 64-bit PAR.
2053          */
2054         par64 = (1 << 11); /* LPAE bit always set */
2055         if (!ret) {
2056             par64 |= phys_addr & ~0xfffULL;
2057             if (!attrs.secure) {
2058                 par64 |= (1 << 9); /* NS */
2059             }
2060             /* We don't set the ATTR or SH fields in the PAR. */
2061         } else {
2062             par64 |= 1; /* F */
2063             par64 |= (fsr & 0x3f) << 1; /* FS */
2064             /* Note that S2WLK and FSTAGE are always zero, because we don't
2065              * implement virtualization and therefore there can't be a stage 2
2066              * fault.
2067              */
2068         }
2069     } else {
2070         /* fsr is a DFSR/IFSR value for the short descriptor
2071          * translation table format (with WnR always clear).
2072          * Convert it to a 32-bit PAR.
2073          */
2074         if (!ret) {
2075             /* We do not set any attribute bits in the PAR */
2076             if (page_size == (1 << 24)
2077                 && arm_feature(env, ARM_FEATURE_V7)) {
2078                 par64 = (phys_addr & 0xff000000) | (1 << 1);
2079             } else {
2080                 par64 = phys_addr & 0xfffff000;
2081             }
2082             if (!attrs.secure) {
2083                 par64 |= (1 << 9); /* NS */
2084             }
2085         } else {
2086             par64 = ((fsr & (1 << 10)) >> 5) | ((fsr & (1 << 12)) >> 6) |
2087                     ((fsr & 0xf) << 1) | 1;
2088         }
2089     }
2090     return par64;
2091 }
2092
2093 static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
2094 {
2095     int access_type = ri->opc2 & 1;
2096     uint64_t par64;
2097     ARMMMUIdx mmu_idx;
2098     int el = arm_current_el(env);
2099     bool secure = arm_is_secure_below_el3(env);
2100
2101     switch (ri->opc2 & 6) {
2102     case 0:
2103         /* stage 1 current state PL1: ATS1CPR, ATS1CPW */
2104         switch (el) {
2105         case 3:
2106             mmu_idx = ARMMMUIdx_S1E3;
2107             break;
2108         case 2:
2109             mmu_idx = ARMMMUIdx_S1NSE1;
2110             break;
2111         case 1:
2112             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
2113             break;
2114         default:
2115             g_assert_not_reached();
2116         }
2117         break;
2118     case 2:
2119         /* stage 1 current state PL0: ATS1CUR, ATS1CUW */
2120         switch (el) {
2121         case 3:
2122             mmu_idx = ARMMMUIdx_S1SE0;
2123             break;
2124         case 2:
2125             mmu_idx = ARMMMUIdx_S1NSE0;
2126             break;
2127         case 1:
2128             mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
2129             break;
2130         default:
2131             g_assert_not_reached();
2132         }
2133         break;
2134     case 4:
2135         /* stage 1+2 NonSecure PL1: ATS12NSOPR, ATS12NSOPW */
2136         mmu_idx = ARMMMUIdx_S12NSE1;
2137         break;
2138     case 6:
2139         /* stage 1+2 NonSecure PL0: ATS12NSOUR, ATS12NSOUW */
2140         mmu_idx = ARMMMUIdx_S12NSE0;
2141         break;
2142     default:
2143         g_assert_not_reached();
2144     }
2145
2146     par64 = do_ats_write(env, value, access_type, mmu_idx);
2147
2148     A32_BANKED_CURRENT_REG_SET(env, par, par64);
2149 }
2150
2151 static void ats1h_write(CPUARMState *env, const ARMCPRegInfo *ri,
2152                         uint64_t value)
2153 {
2154     int access_type = ri->opc2 & 1;
2155     uint64_t par64;
2156
2157     par64 = do_ats_write(env, value, access_type, ARMMMUIdx_S2NS);
2158
2159     A32_BANKED_CURRENT_REG_SET(env, par, par64);
2160 }
2161
2162 static CPAccessResult at_s1e2_access(CPUARMState *env, const ARMCPRegInfo *ri,
2163                                      bool isread)
2164 {
2165     if (arm_current_el(env) == 3 && !(env->cp15.scr_el3 & SCR_NS)) {
2166         return CP_ACCESS_TRAP;
2167     }
2168     return CP_ACCESS_OK;
2169 }
2170
2171 static void ats_write64(CPUARMState *env, const ARMCPRegInfo *ri,
2172                         uint64_t value)
2173 {
2174     int access_type = ri->opc2 & 1;
2175     ARMMMUIdx mmu_idx;
2176     int secure = arm_is_secure_below_el3(env);
2177
2178     switch (ri->opc2 & 6) {
2179     case 0:
2180         switch (ri->opc1) {
2181         case 0: /* AT S1E1R, AT S1E1W */
2182             mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S1NSE1;
2183             break;
2184         case 4: /* AT S1E2R, AT S1E2W */
2185             mmu_idx = ARMMMUIdx_S1E2;
2186             break;
2187         case 6: /* AT S1E3R, AT S1E3W */
2188             mmu_idx = ARMMMUIdx_S1E3;
2189             break;
2190         default:
2191             g_assert_not_reached();
2192         }
2193         break;
2194     case 2: /* AT S1E0R, AT S1E0W */
2195         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S1NSE0;
2196         break;
2197     case 4: /* AT S12E1R, AT S12E1W */
2198         mmu_idx = secure ? ARMMMUIdx_S1SE1 : ARMMMUIdx_S12NSE1;
2199         break;
2200     case 6: /* AT S12E0R, AT S12E0W */
2201         mmu_idx = secure ? ARMMMUIdx_S1SE0 : ARMMMUIdx_S12NSE0;
2202         break;
2203     default:
2204         g_assert_not_reached();
2205     }
2206
2207     env->cp15.par_el[1] = do_ats_write(env, value, access_type, mmu_idx);
2208 }
2209 #endif
2210
2211 static const ARMCPRegInfo vapa_cp_reginfo[] = {
2212     { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
2213       .access = PL1_RW, .resetvalue = 0,
2214       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
2215                              offsetoflow32(CPUARMState, cp15.par_ns) },
2216       .writefn = par_write },
2217 #ifndef CONFIG_USER_ONLY
2218     /* This underdecoding is safe because the reginfo is NO_RAW. */
2219     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
2220       .access = PL1_W, .accessfn = ats_access,
2221       .writefn = ats_write, .type = ARM_CP_NO_RAW },
2222 #endif
2223     REGINFO_SENTINEL
2224 };
2225
2226 /* Return basic MPU access permission bits.  */
2227 static uint32_t simple_mpu_ap_bits(uint32_t val)
2228 {
2229     uint32_t ret;
2230     uint32_t mask;
2231     int i;
2232     ret = 0;
2233     mask = 3;
2234     for (i = 0; i < 16; i += 2) {
2235         ret |= (val >> i) & mask;
2236         mask <<= 2;
2237     }
2238     return ret;
2239 }
2240
2241 /* Pad basic MPU access permission bits to extended format.  */
2242 static uint32_t extended_mpu_ap_bits(uint32_t val)
2243 {
2244     uint32_t ret;
2245     uint32_t mask;
2246     int i;
2247     ret = 0;
2248     mask = 3;
2249     for (i = 0; i < 16; i += 2) {
2250         ret |= (val & mask) << i;
2251         mask <<= 2;
2252     }
2253     return ret;
2254 }
2255
2256 static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
2257                                  uint64_t value)
2258 {
2259     env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
2260 }
2261
2262 static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
2263 {
2264     return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
2265 }
2266
2267 static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
2268                                  uint64_t value)
2269 {
2270     env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
2271 }
2272
2273 static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
2274 {
2275     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
2276 }
2277
2278 static uint64_t pmsav7_read(CPUARMState *env, const ARMCPRegInfo *ri)
2279 {
2280     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
2281
2282     if (!u32p) {
2283         return 0;
2284     }
2285
2286     u32p += env->cp15.c6_rgnr;
2287     return *u32p;
2288 }
2289
2290 static void pmsav7_write(CPUARMState *env, const ARMCPRegInfo *ri,
2291                          uint64_t value)
2292 {
2293     ARMCPU *cpu = arm_env_get_cpu(env);
2294     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
2295
2296     if (!u32p) {
2297         return;
2298     }
2299
2300     u32p += env->cp15.c6_rgnr;
2301     tlb_flush(CPU(cpu), 1); /* Mappings may have changed - purge! */
2302     *u32p = value;
2303 }
2304
2305 static void pmsav7_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2306 {
2307     ARMCPU *cpu = arm_env_get_cpu(env);
2308     uint32_t *u32p = *(uint32_t **)raw_ptr(env, ri);
2309
2310     if (!u32p) {
2311         return;
2312     }
2313
2314     memset(u32p, 0, sizeof(*u32p) * cpu->pmsav7_dregion);
2315 }
2316
2317 static void pmsav7_rgnr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2318                               uint64_t value)
2319 {
2320     ARMCPU *cpu = arm_env_get_cpu(env);
2321     uint32_t nrgs = cpu->pmsav7_dregion;
2322
2323     if (value >= nrgs) {
2324         qemu_log_mask(LOG_GUEST_ERROR,
2325                       "PMSAv7 RGNR write >= # supported regions, %" PRIu32
2326                       " > %" PRIu32 "\n", (uint32_t)value, nrgs);
2327         return;
2328     }
2329
2330     raw_write(env, ri, value);
2331 }
2332
2333 static const ARMCPRegInfo pmsav7_cp_reginfo[] = {
2334     { .name = "DRBAR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 0,
2335       .access = PL1_RW, .type = ARM_CP_NO_RAW,
2336       .fieldoffset = offsetof(CPUARMState, pmsav7.drbar),
2337       .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
2338     { .name = "DRSR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 2,
2339       .access = PL1_RW, .type = ARM_CP_NO_RAW,
2340       .fieldoffset = offsetof(CPUARMState, pmsav7.drsr),
2341       .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
2342     { .name = "DRACR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 1, .opc2 = 4,
2343       .access = PL1_RW, .type = ARM_CP_NO_RAW,
2344       .fieldoffset = offsetof(CPUARMState, pmsav7.dracr),
2345       .readfn = pmsav7_read, .writefn = pmsav7_write, .resetfn = pmsav7_reset },
2346     { .name = "RGNR", .cp = 15, .crn = 6, .opc1 = 0, .crm = 2, .opc2 = 0,
2347       .access = PL1_RW,
2348       .fieldoffset = offsetof(CPUARMState, cp15.c6_rgnr),
2349       .writefn = pmsav7_rgnr_write },
2350     REGINFO_SENTINEL
2351 };
2352
2353 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
2354     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
2355       .access = PL1_RW, .type = ARM_CP_ALIAS,
2356       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
2357       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
2358     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
2359       .access = PL1_RW, .type = ARM_CP_ALIAS,
2360       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
2361       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
2362     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
2363       .access = PL1_RW,
2364       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
2365       .resetvalue = 0, },
2366     { .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
2367       .access = PL1_RW,
2368       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
2369       .resetvalue = 0, },
2370     { .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
2371       .access = PL1_RW,
2372       .fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
2373     { .name = "ICACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
2374       .access = PL1_RW,
2375       .fieldoffset = offsetof(CPUARMState, cp15.c2_insn), .resetvalue = 0, },
2376     /* Protection region base and size registers */
2377     { .name = "946_PRBS0", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0,
2378       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2379       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[0]) },
2380     { .name = "946_PRBS1", .cp = 15, .crn = 6, .crm = 1, .opc1 = 0,
2381       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2382       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[1]) },
2383     { .name = "946_PRBS2", .cp = 15, .crn = 6, .crm = 2, .opc1 = 0,
2384       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2385       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[2]) },
2386     { .name = "946_PRBS3", .cp = 15, .crn = 6, .crm = 3, .opc1 = 0,
2387       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2388       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[3]) },
2389     { .name = "946_PRBS4", .cp = 15, .crn = 6, .crm = 4, .opc1 = 0,
2390       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2391       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[4]) },
2392     { .name = "946_PRBS5", .cp = 15, .crn = 6, .crm = 5, .opc1 = 0,
2393       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2394       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[5]) },
2395     { .name = "946_PRBS6", .cp = 15, .crn = 6, .crm = 6, .opc1 = 0,
2396       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2397       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[6]) },
2398     { .name = "946_PRBS7", .cp = 15, .crn = 6, .crm = 7, .opc1 = 0,
2399       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
2400       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[7]) },
2401     REGINFO_SENTINEL
2402 };
2403
2404 static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
2405                                  uint64_t value)
2406 {
2407     TCR *tcr = raw_ptr(env, ri);
2408     int maskshift = extract32(value, 0, 3);
2409
2410     if (!arm_feature(env, ARM_FEATURE_V8)) {
2411         if (arm_feature(env, ARM_FEATURE_LPAE) && (value & TTBCR_EAE)) {
2412             /* Pre ARMv8 bits [21:19], [15:14] and [6:3] are UNK/SBZP when
2413              * using Long-desciptor translation table format */
2414             value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
2415         } else if (arm_feature(env, ARM_FEATURE_EL3)) {
2416             /* In an implementation that includes the Security Extensions
2417              * TTBCR has additional fields PD0 [4] and PD1 [5] for
2418              * Short-descriptor translation table format.
2419              */
2420             value &= TTBCR_PD1 | TTBCR_PD0 | TTBCR_N;
2421         } else {
2422             value &= TTBCR_N;
2423         }
2424     }
2425
2426     /* Update the masks corresponding to the TCR bank being written
2427      * Note that we always calculate mask and base_mask, but
2428      * they are only used for short-descriptor tables (ie if EAE is 0);
2429      * for long-descriptor tables the TCR fields are used differently
2430      * and the mask and base_mask values are meaningless.
2431      */
2432     tcr->raw_tcr = value;
2433     tcr->mask = ~(((uint32_t)0xffffffffu) >> maskshift);
2434     tcr->base_mask = ~((uint32_t)0x3fffu >> maskshift);
2435 }
2436
2437 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2438                              uint64_t value)
2439 {
2440     ARMCPU *cpu = arm_env_get_cpu(env);
2441
2442     if (arm_feature(env, ARM_FEATURE_LPAE)) {
2443         /* With LPAE the TTBCR could result in a change of ASID
2444          * via the TTBCR.A1 bit, so do a TLB flush.
2445          */
2446         tlb_flush(CPU(cpu), 1);
2447     }
2448     vmsa_ttbcr_raw_write(env, ri, value);
2449 }
2450
2451 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
2452 {
2453     TCR *tcr = raw_ptr(env, ri);
2454
2455     /* Reset both the TCR as well as the masks corresponding to the bank of
2456      * the TCR being reset.
2457      */
2458     tcr->raw_tcr = 0;
2459     tcr->mask = 0;
2460     tcr->base_mask = 0xffffc000u;
2461 }
2462
2463 static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2464                                uint64_t value)
2465 {
2466     ARMCPU *cpu = arm_env_get_cpu(env);
2467     TCR *tcr = raw_ptr(env, ri);
2468
2469     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
2470     tlb_flush(CPU(cpu), 1);
2471     tcr->raw_tcr = value;
2472 }
2473
2474 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2475                             uint64_t value)
2476 {
2477     /* 64 bit accesses to the TTBRs can change the ASID and so we
2478      * must flush the TLB.
2479      */
2480     if (cpreg_field_is_64bit(ri)) {
2481         ARMCPU *cpu = arm_env_get_cpu(env);
2482
2483         tlb_flush(CPU(cpu), 1);
2484     }
2485     raw_write(env, ri, value);
2486 }
2487
2488 static void vttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2489                         uint64_t value)
2490 {
2491     ARMCPU *cpu = arm_env_get_cpu(env);
2492     CPUState *cs = CPU(cpu);
2493
2494     /* Accesses to VTTBR may change the VMID so we must flush the TLB.  */
2495     if (raw_read(env, ri) != value) {
2496         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
2497                             ARMMMUIdx_S2NS, -1);
2498         raw_write(env, ri, value);
2499     }
2500 }
2501
2502 static const ARMCPRegInfo vmsa_pmsa_cp_reginfo[] = {
2503     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
2504       .access = PL1_RW, .type = ARM_CP_ALIAS,
2505       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
2506                              offsetoflow32(CPUARMState, cp15.dfsr_ns) }, },
2507     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
2508       .access = PL1_RW, .resetvalue = 0,
2509       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
2510                              offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
2511     { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
2512       .access = PL1_RW, .resetvalue = 0,
2513       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
2514                              offsetof(CPUARMState, cp15.dfar_ns) } },
2515     { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
2516       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
2517       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
2518       .resetvalue = 0, },
2519     REGINFO_SENTINEL
2520 };
2521
2522 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
2523     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
2524       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
2525       .access = PL1_RW,
2526       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
2527     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
2528       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
2529       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
2530       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
2531                              offsetof(CPUARMState, cp15.ttbr0_ns) } },
2532     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
2533       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
2534       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
2535       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
2536                              offsetof(CPUARMState, cp15.ttbr1_ns) } },
2537     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
2538       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
2539       .access = PL1_RW, .writefn = vmsa_tcr_el1_write,
2540       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
2541       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
2542     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
2543       .access = PL1_RW, .type = ARM_CP_ALIAS, .writefn = vmsa_ttbcr_write,
2544       .raw_writefn = vmsa_ttbcr_raw_write,
2545       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
2546                              offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
2547     REGINFO_SENTINEL
2548 };
2549
2550 static void omap_ticonfig_write(CPUARMState *env, const ARMCPRegInfo *ri,
2551                                 uint64_t value)
2552 {
2553     env->cp15.c15_ticonfig = value & 0xe7;
2554     /* The OS_TYPE bit in this register changes the reported CPUID! */
2555     env->cp15.c0_cpuid = (value & (1 << 5)) ?
2556         ARM_CPUID_TI915T : ARM_CPUID_TI925T;
2557 }
2558
2559 static void omap_threadid_write(CPUARMState *env, const ARMCPRegInfo *ri,
2560                                 uint64_t value)
2561 {
2562     env->cp15.c15_threadid = value & 0xffff;
2563 }
2564
2565 static void omap_wfi_write(CPUARMState *env, const ARMCPRegInfo *ri,
2566                            uint64_t value)
2567 {
2568     /* Wait-for-interrupt (deprecated) */
2569     cpu_interrupt(CPU(arm_env_get_cpu(env)), CPU_INTERRUPT_HALT);
2570 }
2571
2572 static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
2573                                   uint64_t value)
2574 {
2575     /* On OMAP there are registers indicating the max/min index of dcache lines
2576      * containing a dirty line; cache flush operations have to reset these.
2577      */
2578     env->cp15.c15_i_max = 0x000;
2579     env->cp15.c15_i_min = 0xff0;
2580 }
2581
2582 static const ARMCPRegInfo omap_cp_reginfo[] = {
2583     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
2584       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
2585       .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
2586       .resetvalue = 0, },
2587     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
2588       .access = PL1_RW, .type = ARM_CP_NOP },
2589     { .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
2590       .access = PL1_RW,
2591       .fieldoffset = offsetof(CPUARMState, cp15.c15_ticonfig), .resetvalue = 0,
2592       .writefn = omap_ticonfig_write },
2593     { .name = "IMAX", .cp = 15, .crn = 15, .crm = 2, .opc1 = 0, .opc2 = 0,
2594       .access = PL1_RW,
2595       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_max), .resetvalue = 0, },
2596     { .name = "IMIN", .cp = 15, .crn = 15, .crm = 3, .opc1 = 0, .opc2 = 0,
2597       .access = PL1_RW, .resetvalue = 0xff0,
2598       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_min) },
2599     { .name = "THREADID", .cp = 15, .crn = 15, .crm = 4, .opc1 = 0, .opc2 = 0,
2600       .access = PL1_RW,
2601       .fieldoffset = offsetof(CPUARMState, cp15.c15_threadid), .resetvalue = 0,
2602       .writefn = omap_threadid_write },
2603     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
2604       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
2605       .type = ARM_CP_NO_RAW,
2606       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
2607     /* TODO: Peripheral port remap register:
2608      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
2609      * base address at $rn & ~0xfff and map size of 0x200 << ($rn & 0xfff),
2610      * when MMU is off.
2611      */
2612     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
2613       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
2614       .type = ARM_CP_OVERRIDE | ARM_CP_NO_RAW,
2615       .writefn = omap_cachemaint_write },
2616     { .name = "C9", .cp = 15, .crn = 9,
2617       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
2618       .type = ARM_CP_CONST | ARM_CP_OVERRIDE, .resetvalue = 0 },
2619     REGINFO_SENTINEL
2620 };
2621
2622 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
2623                               uint64_t value)
2624 {
2625     env->cp15.c15_cpar = value & 0x3fff;
2626 }
2627
2628 static const ARMCPRegInfo xscale_cp_reginfo[] = {
2629     { .name = "XSCALE_CPAR",
2630       .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
2631       .fieldoffset = offsetof(CPUARMState, cp15.c15_cpar), .resetvalue = 0,
2632       .writefn = xscale_cpar_write, },
2633     { .name = "XSCALE_AUXCR",
2634       .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW,
2635       .fieldoffset = offsetof(CPUARMState, cp15.c1_xscaleauxcr),
2636       .resetvalue = 0, },
2637     /* XScale specific cache-lockdown: since we have no cache we NOP these
2638      * and hope the guest does not really rely on cache behaviour.
2639      */
2640     { .name = "XSCALE_LOCK_ICACHE_LINE",
2641       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
2642       .access = PL1_W, .type = ARM_CP_NOP },
2643     { .name = "XSCALE_UNLOCK_ICACHE",
2644       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
2645       .access = PL1_W, .type = ARM_CP_NOP },
2646     { .name = "XSCALE_DCACHE_LOCK",
2647       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 0,
2648       .access = PL1_RW, .type = ARM_CP_NOP },
2649     { .name = "XSCALE_UNLOCK_DCACHE",
2650       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 1,
2651       .access = PL1_W, .type = ARM_CP_NOP },
2652     REGINFO_SENTINEL
2653 };
2654
2655 static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
2656     /* RAZ/WI the whole crn=15 space, when we don't have a more specific
2657      * implementation of this implementation-defined space.
2658      * Ideally this should eventually disappear in favour of actually
2659      * implementing the correct behaviour for all cores.
2660      */
2661     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
2662       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
2663       .access = PL1_RW,
2664       .type = ARM_CP_CONST | ARM_CP_NO_RAW | ARM_CP_OVERRIDE,
2665       .resetvalue = 0 },
2666     REGINFO_SENTINEL
2667 };
2668
2669 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
2670     /* Cache status: RAZ because we have no cache so it's always clean */
2671     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
2672       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2673       .resetvalue = 0 },
2674     REGINFO_SENTINEL
2675 };
2676
2677 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
2678     /* We never have a a block transfer operation in progress */
2679     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
2680       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2681       .resetvalue = 0 },
2682     /* The cache ops themselves: these all NOP for QEMU */
2683     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
2684       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2685     { .name = "IDCR", .cp = 15, .crm = 6, .opc1 = 0,
2686       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2687     { .name = "CDCR", .cp = 15, .crm = 12, .opc1 = 0,
2688       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2689     { .name = "PIR", .cp = 15, .crm = 12, .opc1 = 1,
2690       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2691     { .name = "PDR", .cp = 15, .crm = 12, .opc1 = 2,
2692       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2693     { .name = "CIDCR", .cp = 15, .crm = 14, .opc1 = 0,
2694       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
2695     REGINFO_SENTINEL
2696 };
2697
2698 static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
2699     /* The cache test-and-clean instructions always return (1 << 30)
2700      * to indicate that there are no dirty cache lines.
2701      */
2702     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
2703       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2704       .resetvalue = (1 << 30) },
2705     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
2706       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_RAW,
2707       .resetvalue = (1 << 30) },
2708     REGINFO_SENTINEL
2709 };
2710
2711 static const ARMCPRegInfo strongarm_cp_reginfo[] = {
2712     /* Ignore ReadBuffer accesses */
2713     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
2714       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
2715       .access = PL1_RW, .resetvalue = 0,
2716       .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_RAW },
2717     REGINFO_SENTINEL
2718 };
2719
2720 static uint64_t midr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2721 {
2722     ARMCPU *cpu = arm_env_get_cpu(env);
2723     unsigned int cur_el = arm_current_el(env);
2724     bool secure = arm_is_secure(env);
2725
2726     if (arm_feature(&cpu->env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
2727         return env->cp15.vpidr_el2;
2728     }
2729     return raw_read(env, ri);
2730 }
2731
2732 static uint64_t mpidr_read_val(CPUARMState *env)
2733 {
2734     ARMCPU *cpu = ARM_CPU(arm_env_get_cpu(env));
2735     uint64_t mpidr = cpu->mp_affinity;
2736
2737     if (arm_feature(env, ARM_FEATURE_V7MP)) {
2738         mpidr |= (1U << 31);
2739         /* Cores which are uniprocessor (non-coherent)
2740          * but still implement the MP extensions set
2741          * bit 30. (For instance, Cortex-R5).
2742          */
2743         if (cpu->mp_is_up) {
2744             mpidr |= (1u << 30);
2745         }
2746     }
2747     return mpidr;
2748 }
2749
2750 static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2751 {
2752     unsigned int cur_el = arm_current_el(env);
2753     bool secure = arm_is_secure(env);
2754
2755     if (arm_feature(env, ARM_FEATURE_EL2) && !secure && cur_el == 1) {
2756         return env->cp15.vmpidr_el2;
2757     }
2758     return mpidr_read_val(env);
2759 }
2760
2761 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
2762     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
2763       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
2764       .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_RAW },
2765     REGINFO_SENTINEL
2766 };
2767
2768 static const ARMCPRegInfo lpae_cp_reginfo[] = {
2769     /* NOP AMAIR0/1 */
2770     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
2771       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
2772       .access = PL1_RW, .type = ARM_CP_CONST,
2773       .resetvalue = 0 },
2774     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
2775     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
2776       .access = PL1_RW, .type = ARM_CP_CONST,
2777       .resetvalue = 0 },
2778     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
2779       .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
2780       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
2781                              offsetof(CPUARMState, cp15.par_ns)} },
2782     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
2783       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2784       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
2785                              offsetof(CPUARMState, cp15.ttbr0_ns) },
2786       .writefn = vmsa_ttbr_write, },
2787     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
2788       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
2789       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
2790                              offsetof(CPUARMState, cp15.ttbr1_ns) },
2791       .writefn = vmsa_ttbr_write, },
2792     REGINFO_SENTINEL
2793 };
2794
2795 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2796 {
2797     return vfp_get_fpcr(env);
2798 }
2799
2800 static void aa64_fpcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2801                             uint64_t value)
2802 {
2803     vfp_set_fpcr(env, value);
2804 }
2805
2806 static uint64_t aa64_fpsr_read(CPUARMState *env, const ARMCPRegInfo *ri)
2807 {
2808     return vfp_get_fpsr(env);
2809 }
2810
2811 static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2812                             uint64_t value)
2813 {
2814     vfp_set_fpsr(env, value);
2815 }
2816
2817 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri,
2818                                        bool isread)
2819 {
2820     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
2821         return CP_ACCESS_TRAP;
2822     }
2823     return CP_ACCESS_OK;
2824 }
2825
2826 static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
2827                             uint64_t value)
2828 {
2829     env->daif = value & PSTATE_DAIF;
2830 }
2831
2832 static CPAccessResult aa64_cacheop_access(CPUARMState *env,
2833                                           const ARMCPRegInfo *ri,
2834                                           bool isread)
2835 {
2836     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
2837      * SCTLR_EL1.UCI is set.
2838      */
2839     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) {
2840         return CP_ACCESS_TRAP;
2841     }
2842     return CP_ACCESS_OK;
2843 }
2844
2845 /* See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
2846  * Page D4-1736 (DDI0487A.b)
2847  */
2848
2849 static void tlbi_aa64_vmalle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2850                                     uint64_t value)
2851 {
2852     ARMCPU *cpu = arm_env_get_cpu(env);
2853     CPUState *cs = CPU(cpu);
2854
2855     if (arm_is_secure_below_el3(env)) {
2856         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2857     } else {
2858         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
2859     }
2860 }
2861
2862 static void tlbi_aa64_vmalle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2863                                       uint64_t value)
2864 {
2865     bool sec = arm_is_secure_below_el3(env);
2866     CPUState *other_cs;
2867
2868     CPU_FOREACH(other_cs) {
2869         if (sec) {
2870             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2871         } else {
2872             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
2873                                 ARMMMUIdx_S12NSE0, -1);
2874         }
2875     }
2876 }
2877
2878 static void tlbi_aa64_alle1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2879                                   uint64_t value)
2880 {
2881     /* Note that the 'ALL' scope must invalidate both stage 1 and
2882      * stage 2 translations, whereas most other scopes only invalidate
2883      * stage 1 translations.
2884      */
2885     ARMCPU *cpu = arm_env_get_cpu(env);
2886     CPUState *cs = CPU(cpu);
2887
2888     if (arm_is_secure_below_el3(env)) {
2889         tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2890     } else {
2891         if (arm_feature(env, ARM_FEATURE_EL2)) {
2892             tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0,
2893                                 ARMMMUIdx_S2NS, -1);
2894         } else {
2895             tlb_flush_by_mmuidx(cs, ARMMMUIdx_S12NSE1, ARMMMUIdx_S12NSE0, -1);
2896         }
2897     }
2898 }
2899
2900 static void tlbi_aa64_alle2_write(CPUARMState *env, const ARMCPRegInfo *ri,
2901                                   uint64_t value)
2902 {
2903     ARMCPU *cpu = arm_env_get_cpu(env);
2904     CPUState *cs = CPU(cpu);
2905
2906     tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E2, -1);
2907 }
2908
2909 static void tlbi_aa64_alle3_write(CPUARMState *env, const ARMCPRegInfo *ri,
2910                                   uint64_t value)
2911 {
2912     ARMCPU *cpu = arm_env_get_cpu(env);
2913     CPUState *cs = CPU(cpu);
2914
2915     tlb_flush_by_mmuidx(cs, ARMMMUIdx_S1E3, -1);
2916 }
2917
2918 static void tlbi_aa64_alle1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2919                                     uint64_t value)
2920 {
2921     /* Note that the 'ALL' scope must invalidate both stage 1 and
2922      * stage 2 translations, whereas most other scopes only invalidate
2923      * stage 1 translations.
2924      */
2925     bool sec = arm_is_secure_below_el3(env);
2926     bool has_el2 = arm_feature(env, ARM_FEATURE_EL2);
2927     CPUState *other_cs;
2928
2929     CPU_FOREACH(other_cs) {
2930         if (sec) {
2931             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1SE1, ARMMMUIdx_S1SE0, -1);
2932         } else if (has_el2) {
2933             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
2934                                 ARMMMUIdx_S12NSE0, ARMMMUIdx_S2NS, -1);
2935         } else {
2936             tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S12NSE1,
2937                                 ARMMMUIdx_S12NSE0, -1);
2938         }
2939     }
2940 }
2941
2942 static void tlbi_aa64_alle2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2943                                     uint64_t value)
2944 {
2945     CPUState *other_cs;
2946
2947     CPU_FOREACH(other_cs) {
2948         tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E2, -1);
2949     }
2950 }
2951
2952 static void tlbi_aa64_alle3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2953                                     uint64_t value)
2954 {
2955     CPUState *other_cs;
2956
2957     CPU_FOREACH(other_cs) {
2958         tlb_flush_by_mmuidx(other_cs, ARMMMUIdx_S1E3, -1);
2959     }
2960 }
2961
2962 static void tlbi_aa64_vae1_write(CPUARMState *env, const ARMCPRegInfo *ri,
2963                                  uint64_t value)
2964 {
2965     /* Invalidate by VA, EL1&0 (AArch64 version).
2966      * Currently handles all of VAE1, VAAE1, VAALE1 and VALE1,
2967      * since we don't support flush-for-specific-ASID-only or
2968      * flush-last-level-only.
2969      */
2970     ARMCPU *cpu = arm_env_get_cpu(env);
2971     CPUState *cs = CPU(cpu);
2972     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2973
2974     if (arm_is_secure_below_el3(env)) {
2975         tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1SE1,
2976                                  ARMMMUIdx_S1SE0, -1);
2977     } else {
2978         tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S12NSE1,
2979                                  ARMMMUIdx_S12NSE0, -1);
2980     }
2981 }
2982
2983 static void tlbi_aa64_vae2_write(CPUARMState *env, const ARMCPRegInfo *ri,
2984                                  uint64_t value)
2985 {
2986     /* Invalidate by VA, EL2
2987      * Currently handles both VAE2 and VALE2, since we don't support
2988      * flush-last-level-only.
2989      */
2990     ARMCPU *cpu = arm_env_get_cpu(env);
2991     CPUState *cs = CPU(cpu);
2992     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2993
2994     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E2, -1);
2995 }
2996
2997 static void tlbi_aa64_vae3_write(CPUARMState *env, const ARMCPRegInfo *ri,
2998                                  uint64_t value)
2999 {
3000     /* Invalidate by VA, EL3
3001      * Currently handles both VAE3 and VALE3, since we don't support
3002      * flush-last-level-only.
3003      */
3004     ARMCPU *cpu = arm_env_get_cpu(env);
3005     CPUState *cs = CPU(cpu);
3006     uint64_t pageaddr = sextract64(value << 12, 0, 56);
3007
3008     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S1E3, -1);
3009 }
3010
3011 static void tlbi_aa64_vae1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
3012                                    uint64_t value)
3013 {
3014     bool sec = arm_is_secure_below_el3(env);
3015     CPUState *other_cs;
3016     uint64_t pageaddr = sextract64(value << 12, 0, 56);
3017
3018     CPU_FOREACH(other_cs) {
3019         if (sec) {
3020             tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1SE1,
3021                                      ARMMMUIdx_S1SE0, -1);
3022         } else {
3023             tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S12NSE1,
3024                                      ARMMMUIdx_S12NSE0, -1);
3025         }
3026     }
3027 }
3028
3029 static void tlbi_aa64_vae2is_write(CPUARMState *env, const ARMCPRegInfo *ri,
3030                                    uint64_t value)
3031 {
3032     CPUState *other_cs;
3033     uint64_t pageaddr = sextract64(value << 12, 0, 56);
3034
3035     CPU_FOREACH(other_cs) {
3036         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E2, -1);
3037     }
3038 }
3039
3040 static void tlbi_aa64_vae3is_write(CPUARMState *env, const ARMCPRegInfo *ri,
3041                                    uint64_t value)
3042 {
3043     CPUState *other_cs;
3044     uint64_t pageaddr = sextract64(value << 12, 0, 56);
3045
3046     CPU_FOREACH(other_cs) {
3047         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S1E3, -1);
3048     }
3049 }
3050
3051 static void tlbi_aa64_ipas2e1_write(CPUARMState *env, const ARMCPRegInfo *ri,
3052                                     uint64_t value)
3053 {
3054     /* Invalidate by IPA. This has to invalidate any structures that
3055      * contain only stage 2 translation information, but does not need
3056      * to apply to structures that contain combined stage 1 and stage 2
3057      * translation information.
3058      * This must NOP if EL2 isn't implemented or SCR_EL3.NS is zero.
3059      */
3060     ARMCPU *cpu = arm_env_get_cpu(env);
3061     CPUState *cs = CPU(cpu);
3062     uint64_t pageaddr;
3063
3064     if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
3065         return;
3066     }
3067
3068     pageaddr = sextract64(value << 12, 0, 48);
3069
3070     tlb_flush_page_by_mmuidx(cs, pageaddr, ARMMMUIdx_S2NS, -1);
3071 }
3072
3073 static void tlbi_aa64_ipas2e1is_write(CPUARMState *env, const ARMCPRegInfo *ri,
3074                                       uint64_t value)
3075 {
3076     CPUState *other_cs;
3077     uint64_t pageaddr;
3078
3079     if (!arm_feature(env, ARM_FEATURE_EL2) || !(env->cp15.scr_el3 & SCR_NS)) {
3080         return;
3081     }
3082
3083     pageaddr = sextract64(value << 12, 0, 48);
3084
3085     CPU_FOREACH(other_cs) {
3086         tlb_flush_page_by_mmuidx(other_cs, pageaddr, ARMMMUIdx_S2NS, -1);
3087     }
3088 }
3089
3090 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri,
3091                                       bool isread)
3092 {
3093     /* We don't implement EL2, so the only control on DC ZVA is the
3094      * bit in the SCTLR which can prohibit access for EL0.
3095      */
3096     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
3097         return CP_ACCESS_TRAP;
3098     }
3099     return CP_ACCESS_OK;
3100 }
3101
3102 static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
3103 {
3104     ARMCPU *cpu = arm_env_get_cpu(env);
3105     int dzp_bit = 1 << 4;
3106
3107     /* DZP indicates whether DC ZVA access is allowed */
3108     if (aa64_zva_access(env, NULL, false) == CP_ACCESS_OK) {
3109         dzp_bit = 0;
3110     }
3111     return cpu->dcz_blocksize | dzp_bit;
3112 }
3113
3114 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
3115                                     bool isread)
3116 {
3117     if (!(env->pstate & PSTATE_SP)) {
3118         /* Access to SP_EL0 is undefined if it's being used as
3119          * the stack pointer.
3120          */
3121         return CP_ACCESS_TRAP_UNCATEGORIZED;
3122     }
3123     return CP_ACCESS_OK;
3124 }
3125
3126 static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
3127 {
3128     return env->pstate & PSTATE_SP;
3129 }
3130
3131 static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
3132 {
3133     update_spsel(env, val);
3134 }
3135
3136 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3137                         uint64_t value)
3138 {
3139     ARMCPU *cpu = arm_env_get_cpu(env);
3140
3141     if (raw_read(env, ri) == value) {
3142         /* Skip the TLB flush if nothing actually changed; Linux likes
3143          * to do a lot of pointless SCTLR writes.
3144          */
3145         return;
3146     }
3147
3148     raw_write(env, ri, value);
3149     /* ??? Lots of these bits are not implemented.  */
3150     /* This may enable/disable the MMU, so do a TLB flush.  */
3151     tlb_flush(CPU(cpu), 1);
3152 }
3153
3154 static CPAccessResult fpexc32_access(CPUARMState *env, const ARMCPRegInfo *ri,
3155                                      bool isread)
3156 {
3157     if ((env->cp15.cptr_el[2] & CPTR_TFP) && arm_current_el(env) == 2) {
3158         return CP_ACCESS_TRAP_FP_EL2;
3159     }
3160     if (env->cp15.cptr_el[3] & CPTR_TFP) {
3161         return CP_ACCESS_TRAP_FP_EL3;
3162     }
3163     return CP_ACCESS_OK;
3164 }
3165
3166 static void sdcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
3167                        uint64_t value)
3168 {
3169     env->cp15.mdcr_el3 = value & SDCR_VALID_MASK;
3170 }
3171
3172 static const ARMCPRegInfo v8_cp_reginfo[] = {
3173     /* Minimal set of EL0-visible registers. This will need to be expanded
3174      * significantly for system emulation of AArch64 CPUs.
3175      */
3176     { .name = "NZCV", .state = ARM_CP_STATE_AA64,
3177       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
3178       .access = PL0_RW, .type = ARM_CP_NZCV },
3179     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
3180       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
3181       .type = ARM_CP_NO_RAW,
3182       .access = PL0_RW, .accessfn = aa64_daif_access,
3183       .fieldoffset = offsetof(CPUARMState, daif),
3184       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
3185     { .name = "FPCR", .state = ARM_CP_STATE_AA64,
3186       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
3187       .access = PL0_RW, .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
3188     { .name = "FPSR", .state = ARM_CP_STATE_AA64,
3189       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
3190       .access = PL0_RW, .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
3191     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
3192       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
3193       .access = PL0_R, .type = ARM_CP_NO_RAW,
3194       .readfn = aa64_dczid_read },
3195     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
3196       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
3197       .access = PL0_W, .type = ARM_CP_DC_ZVA,
3198 #ifndef CONFIG_USER_ONLY
3199       /* Avoid overhead of an access check that always passes in user-mode */
3200       .accessfn = aa64_zva_access,
3201 #endif
3202     },
3203     { .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
3204       .opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
3205       .access = PL1_R, .type = ARM_CP_CURRENTEL },
3206     /* Cache ops: all NOPs since we don't emulate caches */
3207     { .name = "IC_IALLUIS", .state = ARM_CP_STATE_AA64,
3208       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
3209       .access = PL1_W, .type = ARM_CP_NOP },
3210     { .name = "IC_IALLU", .state = ARM_CP_STATE_AA64,
3211       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
3212       .access = PL1_W, .type = ARM_CP_NOP },
3213     { .name = "IC_IVAU", .state = ARM_CP_STATE_AA64,
3214       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 5, .opc2 = 1,
3215       .access = PL0_W, .type = ARM_CP_NOP,
3216       .accessfn = aa64_cacheop_access },
3217     { .name = "DC_IVAC", .state = ARM_CP_STATE_AA64,
3218       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
3219       .access = PL1_W, .type = ARM_CP_NOP },
3220     { .name = "DC_ISW", .state = ARM_CP_STATE_AA64,
3221       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
3222       .access = PL1_W, .type = ARM_CP_NOP },
3223     { .name = "DC_CVAC", .state = ARM_CP_STATE_AA64,
3224       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 1,
3225       .access = PL0_W, .type = ARM_CP_NOP,
3226       .accessfn = aa64_cacheop_access },
3227     { .name = "DC_CSW", .state = ARM_CP_STATE_AA64,
3228       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
3229       .access = PL1_W, .type = ARM_CP_NOP },
3230     { .name = "DC_CVAU", .state = ARM_CP_STATE_AA64,
3231       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 11, .opc2 = 1,
3232       .access = PL0_W, .type = ARM_CP_NOP,
3233       .accessfn = aa64_cacheop_access },
3234     { .name = "DC_CIVAC", .state = ARM_CP_STATE_AA64,
3235       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 1,
3236       .access = PL0_W, .type = ARM_CP_NOP,
3237       .accessfn = aa64_cacheop_access },
3238     { .name = "DC_CISW", .state = ARM_CP_STATE_AA64,
3239       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
3240       .access = PL1_W, .type = ARM_CP_NOP },
3241     /* TLBI operations */
3242     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
3243       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
3244       .access = PL1_W, .type = ARM_CP_NO_RAW,
3245       .writefn = tlbi_aa64_vmalle1is_write },
3246     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
3247       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
3248       .access = PL1_W, .type = ARM_CP_NO_RAW,
3249       .writefn = tlbi_aa64_vae1is_write },
3250     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
3251       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
3252       .access = PL1_W, .type = ARM_CP_NO_RAW,
3253       .writefn = tlbi_aa64_vmalle1is_write },
3254     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
3255       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
3256       .access = PL1_W, .type = ARM_CP_NO_RAW,
3257       .writefn = tlbi_aa64_vae1is_write },
3258     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
3259       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
3260       .access = PL1_W, .type = ARM_CP_NO_RAW,
3261       .writefn = tlbi_aa64_vae1is_write },
3262     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
3263       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
3264       .access = PL1_W, .type = ARM_CP_NO_RAW,
3265       .writefn = tlbi_aa64_vae1is_write },
3266     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
3267       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
3268       .access = PL1_W, .type = ARM_CP_NO_RAW,
3269       .writefn = tlbi_aa64_vmalle1_write },
3270     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
3271       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
3272       .access = PL1_W, .type = ARM_CP_NO_RAW,
3273       .writefn = tlbi_aa64_vae1_write },
3274     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
3275       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
3276       .access = PL1_W, .type = ARM_CP_NO_RAW,
3277       .writefn = tlbi_aa64_vmalle1_write },
3278     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
3279       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
3280       .access = PL1_W, .type = ARM_CP_NO_RAW,
3281       .writefn = tlbi_aa64_vae1_write },
3282     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
3283       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
3284       .access = PL1_W, .type = ARM_CP_NO_RAW,
3285       .writefn = tlbi_aa64_vae1_write },
3286     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
3287       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
3288       .access = PL1_W, .type = ARM_CP_NO_RAW,
3289       .writefn = tlbi_aa64_vae1_write },
3290     { .name = "TLBI_IPAS2E1IS", .state = ARM_CP_STATE_AA64,
3291       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
3292       .access = PL2_W, .type = ARM_CP_NO_RAW,
3293       .writefn = tlbi_aa64_ipas2e1is_write },
3294     { .name = "TLBI_IPAS2LE1IS", .state = ARM_CP_STATE_AA64,
3295       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
3296       .access = PL2_W, .type = ARM_CP_NO_RAW,
3297       .writefn = tlbi_aa64_ipas2e1is_write },
3298     { .name = "TLBI_ALLE1IS", .state = ARM_CP_STATE_AA64,
3299       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
3300       .access = PL2_W, .type = ARM_CP_NO_RAW,
3301       .writefn = tlbi_aa64_alle1is_write },
3302     { .name = "TLBI_VMALLS12E1IS", .state = ARM_CP_STATE_AA64,
3303       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 6,
3304       .access = PL2_W, .type = ARM_CP_NO_RAW,
3305       .writefn = tlbi_aa64_alle1is_write },
3306     { .name = "TLBI_IPAS2E1", .state = ARM_CP_STATE_AA64,
3307       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
3308       .access = PL2_W, .type = ARM_CP_NO_RAW,
3309       .writefn = tlbi_aa64_ipas2e1_write },
3310     { .name = "TLBI_IPAS2LE1", .state = ARM_CP_STATE_AA64,
3311       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
3312       .access = PL2_W, .type = ARM_CP_NO_RAW,
3313       .writefn = tlbi_aa64_ipas2e1_write },
3314     { .name = "TLBI_ALLE1", .state = ARM_CP_STATE_AA64,
3315       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
3316       .access = PL2_W, .type = ARM_CP_NO_RAW,
3317       .writefn = tlbi_aa64_alle1_write },
3318     { .name = "TLBI_VMALLS12E1", .state = ARM_CP_STATE_AA64,
3319       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 6,
3320       .access = PL2_W, .type = ARM_CP_NO_RAW,
3321       .writefn = tlbi_aa64_alle1is_write },
3322 #ifndef CONFIG_USER_ONLY
3323     /* 64 bit address translation operations */
3324     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
3325       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
3326       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3327     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
3328       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
3329       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3330     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
3331       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
3332       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3333     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
3334       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
3335       .access = PL1_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3336     { .name = "AT_S12E1R", .state = ARM_CP_STATE_AA64,
3337       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 4,
3338       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3339     { .name = "AT_S12E1W", .state = ARM_CP_STATE_AA64,
3340       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 5,
3341       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3342     { .name = "AT_S12E0R", .state = ARM_CP_STATE_AA64,
3343       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 6,
3344       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3345     { .name = "AT_S12E0W", .state = ARM_CP_STATE_AA64,
3346       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 7,
3347       .access = PL2_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3348     /* AT S1E2* are elsewhere as they UNDEF from EL3 if EL2 is not present */
3349     { .name = "AT_S1E3R", .state = ARM_CP_STATE_AA64,
3350       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 0,
3351       .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3352     { .name = "AT_S1E3W", .state = ARM_CP_STATE_AA64,
3353       .opc0 = 1, .opc1 = 6, .crn = 7, .crm = 8, .opc2 = 1,
3354       .access = PL3_W, .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3355     { .name = "PAR_EL1", .state = ARM_CP_STATE_AA64,
3356       .type = ARM_CP_ALIAS,
3357       .opc0 = 3, .opc1 = 0, .crn = 7, .crm = 4, .opc2 = 0,
3358       .access = PL1_RW, .resetvalue = 0,
3359       .fieldoffset = offsetof(CPUARMState, cp15.par_el[1]),
3360       .writefn = par_write },
3361 #endif
3362     /* TLB invalidate last level of translation table walk */
3363     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
3364       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_is_write },
3365     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
3366       .type = ARM_CP_NO_RAW, .access = PL1_W,
3367       .writefn = tlbimvaa_is_write },
3368     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
3369       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimva_write },
3370     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
3371       .type = ARM_CP_NO_RAW, .access = PL1_W, .writefn = tlbimvaa_write },
3372     { .name = "TLBIMVALH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
3373       .type = ARM_CP_NO_RAW, .access = PL2_W,
3374       .writefn = tlbimva_hyp_write },
3375     { .name = "TLBIMVALHIS",
3376       .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
3377       .type = ARM_CP_NO_RAW, .access = PL2_W,
3378       .writefn = tlbimva_hyp_is_write },
3379     { .name = "TLBIIPAS2",
3380       .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 1,
3381       .type = ARM_CP_NO_RAW, .access = PL2_W,
3382       .writefn = tlbiipas2_write },
3383     { .name = "TLBIIPAS2IS",
3384       .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 1,
3385       .type = ARM_CP_NO_RAW, .access = PL2_W,
3386       .writefn = tlbiipas2_is_write },
3387     { .name = "TLBIIPAS2L",
3388       .cp = 15, .opc1 = 4, .crn = 8, .crm = 4, .opc2 = 5,
3389       .type = ARM_CP_NO_RAW, .access = PL2_W,
3390       .writefn = tlbiipas2_write },
3391     { .name = "TLBIIPAS2LIS",
3392       .cp = 15, .opc1 = 4, .crn = 8, .crm = 0, .opc2 = 5,
3393       .type = ARM_CP_NO_RAW, .access = PL2_W,
3394       .writefn = tlbiipas2_is_write },
3395     /* 32 bit cache operations */
3396     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
3397       .type = ARM_CP_NOP, .access = PL1_W },
3398     { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
3399       .type = ARM_CP_NOP, .access = PL1_W },
3400     { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
3401       .type = ARM_CP_NOP, .access = PL1_W },
3402     { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
3403       .type = ARM_CP_NOP, .access = PL1_W },
3404     { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
3405       .type = ARM_CP_NOP, .access = PL1_W },
3406     { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
3407       .type = ARM_CP_NOP, .access = PL1_W },
3408     { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
3409       .type = ARM_CP_NOP, .access = PL1_W },
3410     { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
3411       .type = ARM_CP_NOP, .access = PL1_W },
3412     { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
3413       .type = ARM_CP_NOP, .access = PL1_W },
3414     { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
3415       .type = ARM_CP_NOP, .access = PL1_W },
3416     { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
3417       .type = ARM_CP_NOP, .access = PL1_W },
3418     { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
3419       .type = ARM_CP_NOP, .access = PL1_W },
3420     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
3421       .type = ARM_CP_NOP, .access = PL1_W },
3422     /* MMU Domain access control / MPU write buffer control */
3423     { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
3424       .access = PL1_RW, .resetvalue = 0,
3425       .writefn = dacr_write, .raw_writefn = raw_write,
3426       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
3427                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
3428     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
3429       .type = ARM_CP_ALIAS,
3430       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
3431       .access = PL1_RW,
3432       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
3433     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
3434       .type = ARM_CP_ALIAS,
3435       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
3436       .access = PL1_RW,
3437       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_SVC]) },
3438     /* We rely on the access checks not allowing the guest to write to the
3439      * state field when SPSel indicates that it's being used as the stack
3440      * pointer.
3441      */
3442     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
3443       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
3444       .access = PL1_RW, .accessfn = sp_el0_access,
3445       .type = ARM_CP_ALIAS,
3446       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
3447     { .name = "SP_EL1", .state = ARM_CP_STATE_AA64,
3448       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 1, .opc2 = 0,
3449       .access = PL2_RW, .type = ARM_CP_ALIAS,
3450       .fieldoffset = offsetof(CPUARMState, sp_el[1]) },
3451     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
3452       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
3453       .type = ARM_CP_NO_RAW,
3454       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
3455     { .name = "FPEXC32_EL2", .state = ARM_CP_STATE_AA64,
3456       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 3, .opc2 = 0,
3457       .type = ARM_CP_ALIAS,
3458       .fieldoffset = offsetof(CPUARMState, vfp.xregs[ARM_VFP_FPEXC]),
3459       .access = PL2_RW, .accessfn = fpexc32_access },
3460     { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
3461       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
3462       .access = PL2_RW, .resetvalue = 0,
3463       .writefn = dacr_write, .raw_writefn = raw_write,
3464       .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
3465     { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
3466       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
3467       .access = PL2_RW, .resetvalue = 0,
3468       .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
3469     { .name = "SPSR_IRQ", .state = ARM_CP_STATE_AA64,
3470       .type = ARM_CP_ALIAS,
3471       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 0,
3472       .access = PL2_RW,
3473       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_IRQ]) },
3474     { .name = "SPSR_ABT", .state = ARM_CP_STATE_AA64,
3475       .type = ARM_CP_ALIAS,
3476       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 1,
3477       .access = PL2_RW,
3478       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_ABT]) },
3479     { .name = "SPSR_UND", .state = ARM_CP_STATE_AA64,
3480       .type = ARM_CP_ALIAS,
3481       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 2,
3482       .access = PL2_RW,
3483       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_UND]) },
3484     { .name = "SPSR_FIQ", .state = ARM_CP_STATE_AA64,
3485       .type = ARM_CP_ALIAS,
3486       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 3, .opc2 = 3,
3487       .access = PL2_RW,
3488       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_FIQ]) },
3489     { .name = "MDCR_EL3", .state = ARM_CP_STATE_AA64,
3490       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 3, .opc2 = 1,
3491       .resetvalue = 0,
3492       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el3) },
3493     { .name = "SDCR", .type = ARM_CP_ALIAS,
3494       .cp = 15, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 1,
3495       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
3496       .writefn = sdcr_write,
3497       .fieldoffset = offsetoflow32(CPUARMState, cp15.mdcr_el3) },
3498     REGINFO_SENTINEL
3499 };
3500
3501 /* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
3502 static const ARMCPRegInfo el3_no_el2_cp_reginfo[] = {
3503     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
3504       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
3505       .access = PL2_RW,
3506       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
3507     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
3508       .type = ARM_CP_NO_RAW,
3509       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
3510       .access = PL2_RW,
3511       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
3512     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
3513       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
3514       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3515     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
3516       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
3517       .access = PL2_RW, .type = ARM_CP_CONST,
3518       .resetvalue = 0 },
3519     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3520       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
3521       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3522     { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
3523       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
3524       .access = PL2_RW, .type = ARM_CP_CONST,
3525       .resetvalue = 0 },
3526     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3527       .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
3528       .access = PL2_RW, .type = ARM_CP_CONST,
3529       .resetvalue = 0 },
3530     { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
3531       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
3532       .access = PL2_RW, .type = ARM_CP_CONST,
3533       .resetvalue = 0 },
3534     { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
3535       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
3536       .access = PL2_RW, .type = ARM_CP_CONST,
3537       .resetvalue = 0 },
3538     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
3539       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
3540       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3541     { .name = "VTCR_EL2", .state = ARM_CP_STATE_BOTH,
3542       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
3543       .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
3544       .type = ARM_CP_CONST, .resetvalue = 0 },
3545     { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
3546       .cp = 15, .opc1 = 6, .crm = 2,
3547       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3548       .type = ARM_CP_CONST | ARM_CP_64BIT, .resetvalue = 0 },
3549     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
3550       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
3551       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3552     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
3553       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
3554       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3555     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
3556       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
3557       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3558     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
3559       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
3560       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3561     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
3562       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
3563       .resetvalue = 0 },
3564     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
3565       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
3566       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3567     { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
3568       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
3569       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3570     { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
3571       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
3572       .resetvalue = 0 },
3573     { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
3574       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
3575       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3576     { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
3577       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_CONST,
3578       .resetvalue = 0 },
3579     { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
3580       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
3581       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3582     { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
3583       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
3584       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3585     { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
3586       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
3587       .access = PL2_RW, .accessfn = access_tda,
3588       .type = ARM_CP_CONST, .resetvalue = 0 },
3589     { .name = "HPFAR_EL2", .state = ARM_CP_STATE_BOTH,
3590       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
3591       .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
3592       .type = ARM_CP_CONST, .resetvalue = 0 },
3593     { .name = "HSTR_EL2", .state = ARM_CP_STATE_BOTH,
3594       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 3,
3595       .access = PL2_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
3596     REGINFO_SENTINEL
3597 };
3598
3599 static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
3600 {
3601     ARMCPU *cpu = arm_env_get_cpu(env);
3602     uint64_t valid_mask = HCR_MASK;
3603
3604     if (arm_feature(env, ARM_FEATURE_EL3)) {
3605         valid_mask &= ~HCR_HCD;
3606     } else {
3607         valid_mask &= ~HCR_TSC;
3608     }
3609
3610     /* Clear RES0 bits.  */
3611     value &= valid_mask;
3612
3613     /* These bits change the MMU setup:
3614      * HCR_VM enables stage 2 translation
3615      * HCR_PTW forbids certain page-table setups
3616      * HCR_DC Disables stage1 and enables stage2 translation
3617      */
3618     if ((raw_read(env, ri) ^ value) & (HCR_VM | HCR_PTW | HCR_DC)) {
3619         tlb_flush(CPU(cpu), 1);
3620     }
3621     raw_write(env, ri, value);
3622 }
3623
3624 static const ARMCPRegInfo el2_cp_reginfo[] = {
3625     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
3626       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
3627       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
3628       .writefn = hcr_write },
3629     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
3630       .type = ARM_CP_ALIAS,
3631       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
3632       .access = PL2_RW,
3633       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
3634     { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
3635       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
3636       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
3637     { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
3638       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
3639       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
3640     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
3641       .type = ARM_CP_ALIAS,
3642       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
3643       .access = PL2_RW,
3644       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_HYP]) },
3645     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
3646       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
3647       .access = PL2_RW, .writefn = vbar_write,
3648       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
3649       .resetvalue = 0 },
3650     { .name = "SP_EL2", .state = ARM_CP_STATE_AA64,
3651       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 1, .opc2 = 0,
3652       .access = PL3_RW, .type = ARM_CP_ALIAS,
3653       .fieldoffset = offsetof(CPUARMState, sp_el[2]) },
3654     { .name = "CPTR_EL2", .state = ARM_CP_STATE_BOTH,
3655       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 2,
3656       .access = PL2_RW, .accessfn = cptr_access, .resetvalue = 0,
3657       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[2]) },
3658     { .name = "MAIR_EL2", .state = ARM_CP_STATE_BOTH,
3659       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 0,
3660       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[2]),
3661       .resetvalue = 0 },
3662     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3663       .opc1 = 4, .crn = 10, .crm = 2, .opc2 = 1,
3664       .access = PL2_RW, .type = ARM_CP_ALIAS,
3665       .fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el[2]) },
3666     { .name = "AMAIR_EL2", .state = ARM_CP_STATE_BOTH,
3667       .opc0 = 3, .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 0,
3668       .access = PL2_RW, .type = ARM_CP_CONST,
3669       .resetvalue = 0 },
3670     /* HAMAIR1 is mapped to AMAIR_EL2[63:32] */
3671     { .name = "HMAIR1", .state = ARM_CP_STATE_AA32,
3672       .opc1 = 4, .crn = 10, .crm = 3, .opc2 = 1,
3673       .access = PL2_RW, .type = ARM_CP_CONST,
3674       .resetvalue = 0 },
3675     { .name = "AFSR0_EL2", .state = ARM_CP_STATE_BOTH,
3676       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 0,
3677       .access = PL2_RW, .type = ARM_CP_CONST,
3678       .resetvalue = 0 },
3679     { .name = "AFSR1_EL2", .state = ARM_CP_STATE_BOTH,
3680       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 1, .opc2 = 1,
3681       .access = PL2_RW, .type = ARM_CP_CONST,
3682       .resetvalue = 0 },
3683     { .name = "TCR_EL2", .state = ARM_CP_STATE_BOTH,
3684       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 2,
3685       .access = PL2_RW,
3686       /* no .writefn needed as this can't cause an ASID change;
3687        * no .raw_writefn or .resetfn needed as we never use mask/base_mask
3688        */
3689       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[2]) },
3690     { .name = "VTCR", .state = ARM_CP_STATE_AA32,
3691       .cp = 15, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
3692       .type = ARM_CP_ALIAS,
3693       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3694       .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
3695     { .name = "VTCR_EL2", .state = ARM_CP_STATE_AA64,
3696       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 2,
3697       .access = PL2_RW,
3698       /* no .writefn needed as this can't cause an ASID change;
3699        * no .raw_writefn or .resetfn needed as we never use mask/base_mask
3700        */
3701       .fieldoffset = offsetof(CPUARMState, cp15.vtcr_el2) },
3702     { .name = "VTTBR", .state = ARM_CP_STATE_AA32,
3703       .cp = 15, .opc1 = 6, .crm = 2,
3704       .type = ARM_CP_64BIT | ARM_CP_ALIAS,
3705       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3706       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2),
3707       .writefn = vttbr_write },
3708     { .name = "VTTBR_EL2", .state = ARM_CP_STATE_AA64,
3709       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 1, .opc2 = 0,
3710       .access = PL2_RW, .writefn = vttbr_write,
3711       .fieldoffset = offsetof(CPUARMState, cp15.vttbr_el2) },
3712     { .name = "SCTLR_EL2", .state = ARM_CP_STATE_BOTH,
3713       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 0,
3714       .access = PL2_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
3715       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[2]) },
3716     { .name = "TPIDR_EL2", .state = ARM_CP_STATE_BOTH,
3717       .opc0 = 3, .opc1 = 4, .crn = 13, .crm = 0, .opc2 = 2,
3718       .access = PL2_RW, .resetvalue = 0,
3719       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[2]) },
3720     { .name = "TTBR0_EL2", .state = ARM_CP_STATE_AA64,
3721       .opc0 = 3, .opc1 = 4, .crn = 2, .crm = 0, .opc2 = 0,
3722       .access = PL2_RW, .resetvalue = 0,
3723       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
3724     { .name = "HTTBR", .cp = 15, .opc1 = 4, .crm = 2,
3725       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS,
3726       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[2]) },
3727     { .name = "TLBIALLNSNH",
3728       .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 4,
3729       .type = ARM_CP_NO_RAW, .access = PL2_W,
3730       .writefn = tlbiall_nsnh_write },
3731     { .name = "TLBIALLNSNHIS",
3732       .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 4,
3733       .type = ARM_CP_NO_RAW, .access = PL2_W,
3734       .writefn = tlbiall_nsnh_is_write },
3735     { .name = "TLBIALLH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
3736       .type = ARM_CP_NO_RAW, .access = PL2_W,
3737       .writefn = tlbiall_hyp_write },
3738     { .name = "TLBIALLHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
3739       .type = ARM_CP_NO_RAW, .access = PL2_W,
3740       .writefn = tlbiall_hyp_is_write },
3741     { .name = "TLBIMVAH", .cp = 15, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
3742       .type = ARM_CP_NO_RAW, .access = PL2_W,
3743       .writefn = tlbimva_hyp_write },
3744     { .name = "TLBIMVAHIS", .cp = 15, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
3745       .type = ARM_CP_NO_RAW, .access = PL2_W,
3746       .writefn = tlbimva_hyp_is_write },
3747     { .name = "TLBI_ALLE2", .state = ARM_CP_STATE_AA64,
3748       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 0,
3749       .type = ARM_CP_NO_RAW, .access = PL2_W,
3750       .writefn = tlbi_aa64_alle2_write },
3751     { .name = "TLBI_VAE2", .state = ARM_CP_STATE_AA64,
3752       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 1,
3753       .type = ARM_CP_NO_RAW, .access = PL2_W,
3754       .writefn = tlbi_aa64_vae2_write },
3755     { .name = "TLBI_VALE2", .state = ARM_CP_STATE_AA64,
3756       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 7, .opc2 = 5,
3757       .access = PL2_W, .type = ARM_CP_NO_RAW,
3758       .writefn = tlbi_aa64_vae2_write },
3759     { .name = "TLBI_ALLE2IS", .state = ARM_CP_STATE_AA64,
3760       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 0,
3761       .access = PL2_W, .type = ARM_CP_NO_RAW,
3762       .writefn = tlbi_aa64_alle2is_write },
3763     { .name = "TLBI_VAE2IS", .state = ARM_CP_STATE_AA64,
3764       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 1,
3765       .type = ARM_CP_NO_RAW, .access = PL2_W,
3766       .writefn = tlbi_aa64_vae2is_write },
3767     { .name = "TLBI_VALE2IS", .state = ARM_CP_STATE_AA64,
3768       .opc0 = 1, .opc1 = 4, .crn = 8, .crm = 3, .opc2 = 5,
3769       .access = PL2_W, .type = ARM_CP_NO_RAW,
3770       .writefn = tlbi_aa64_vae2is_write },
3771 #ifndef CONFIG_USER_ONLY
3772     /* Unlike the other EL2-related AT operations, these must
3773      * UNDEF from EL3 if EL2 is not implemented, which is why we
3774      * define them here rather than with the rest of the AT ops.
3775      */
3776     { .name = "AT_S1E2R", .state = ARM_CP_STATE_AA64,
3777       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
3778       .access = PL2_W, .accessfn = at_s1e2_access,
3779       .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3780     { .name = "AT_S1E2W", .state = ARM_CP_STATE_AA64,
3781       .opc0 = 1, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
3782       .access = PL2_W, .accessfn = at_s1e2_access,
3783       .type = ARM_CP_NO_RAW, .writefn = ats_write64 },
3784     /* The AArch32 ATS1H* operations are CONSTRAINED UNPREDICTABLE
3785      * if EL2 is not implemented; we choose to UNDEF. Behaviour at EL3
3786      * with SCR.NS == 0 outside Monitor mode is UNPREDICTABLE; we choose
3787      * to behave as if SCR.NS was 1.
3788      */
3789     { .name = "ATS1HR", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 0,
3790       .access = PL2_W,
3791       .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
3792     { .name = "ATS1HW", .cp = 15, .opc1 = 4, .crn = 7, .crm = 8, .opc2 = 1,
3793       .access = PL2_W,
3794       .writefn = ats1h_write, .type = ARM_CP_NO_RAW },
3795     { .name = "CNTHCTL_EL2", .state = ARM_CP_STATE_BOTH,
3796       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 1, .opc2 = 0,
3797       /* ARMv7 requires bit 0 and 1 to reset to 1. ARMv8 defines the
3798        * reset values as IMPDEF. We choose to reset to 3 to comply with
3799        * both ARMv7 and ARMv8.
3800        */
3801       .access = PL2_RW, .resetvalue = 3,
3802       .fieldoffset = offsetof(CPUARMState, cp15.cnthctl_el2) },
3803     { .name = "CNTVOFF_EL2", .state = ARM_CP_STATE_AA64,
3804       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 0, .opc2 = 3,
3805       .access = PL2_RW, .type = ARM_CP_IO, .resetvalue = 0,
3806       .writefn = gt_cntvoff_write,
3807       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
3808     { .name = "CNTVOFF", .cp = 15, .opc1 = 4, .crm = 14,
3809       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_ALIAS | ARM_CP_IO,
3810       .writefn = gt_cntvoff_write,
3811       .fieldoffset = offsetof(CPUARMState, cp15.cntvoff_el2) },
3812     { .name = "CNTHP_CVAL_EL2", .state = ARM_CP_STATE_AA64,
3813       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 2,
3814       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
3815       .type = ARM_CP_IO, .access = PL2_RW,
3816       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
3817     { .name = "CNTHP_CVAL", .cp = 15, .opc1 = 6, .crm = 14,
3818       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].cval),
3819       .access = PL2_RW, .type = ARM_CP_64BIT | ARM_CP_IO,
3820       .writefn = gt_hyp_cval_write, .raw_writefn = raw_write },
3821     { .name = "CNTHP_TVAL_EL2", .state = ARM_CP_STATE_BOTH,
3822       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 0,
3823       .type = ARM_CP_NO_RAW | ARM_CP_IO, .access = PL2_RW,
3824       .resetfn = gt_hyp_timer_reset,
3825       .readfn = gt_hyp_tval_read, .writefn = gt_hyp_tval_write },
3826     { .name = "CNTHP_CTL_EL2", .state = ARM_CP_STATE_BOTH,
3827       .type = ARM_CP_IO,
3828       .opc0 = 3, .opc1 = 4, .crn = 14, .crm = 2, .opc2 = 1,
3829       .access = PL2_RW,
3830       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_HYP].ctl),
3831       .resetvalue = 0,
3832       .writefn = gt_hyp_ctl_write, .raw_writefn = raw_write },
3833 #endif
3834     /* The only field of MDCR_EL2 that has a defined architectural reset value
3835      * is MDCR_EL2.HPMN which should reset to the value of PMCR_EL0.N; but we
3836      * don't impelment any PMU event counters, so using zero as a reset
3837      * value for MDCR_EL2 is okay
3838      */
3839     { .name = "MDCR_EL2", .state = ARM_CP_STATE_BOTH,
3840       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 1,
3841       .access = PL2_RW, .resetvalue = 0,
3842       .fieldoffset = offsetof(CPUARMState, cp15.mdcr_el2), },
3843     { .name = "HPFAR", .state = ARM_CP_STATE_AA32,
3844       .cp = 15, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
3845       .access = PL2_RW, .accessfn = access_el3_aa32ns,
3846       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
3847     { .name = "HPFAR_EL2", .state = ARM_CP_STATE_AA64,
3848       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 4,
3849       .access = PL2_RW,
3850       .fieldoffset = offsetof(CPUARMState, cp15.hpfar_el2) },
3851     { .name = "HSTR_EL2", .state = ARM_CP_STATE_BOTH,
3852       .cp = 15, .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 3,
3853       .access = PL2_RW,
3854       .fieldoffset = offsetof(CPUARMState, cp15.hstr_el2) },
3855     REGINFO_SENTINEL
3856 };
3857
3858 static CPAccessResult nsacr_access(CPUARMState *env, const ARMCPRegInfo *ri,
3859                                    bool isread)
3860 {
3861     /* The NSACR is RW at EL3, and RO for NS EL1 and NS EL2.
3862      * At Secure EL1 it traps to EL3.
3863      */
3864     if (arm_current_el(env) == 3) {
3865         return CP_ACCESS_OK;
3866     }
3867     if (arm_is_secure_below_el3(env)) {
3868         return CP_ACCESS_TRAP_EL3;
3869     }
3870     /* Accesses from EL1 NS and EL2 NS are UNDEF for write but allow reads. */
3871     if (isread) {
3872         return CP_ACCESS_OK;
3873     }
3874     return CP_ACCESS_TRAP_UNCATEGORIZED;
3875 }
3876
3877 static const ARMCPRegInfo el3_cp_reginfo[] = {
3878     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
3879       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
3880       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
3881       .resetvalue = 0, .writefn = scr_write },
3882     { .name = "SCR",  .type = ARM_CP_ALIAS,
3883       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
3884       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
3885       .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
3886       .writefn = scr_write },
3887     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
3888       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
3889       .access = PL3_RW, .resetvalue = 0,
3890       .fieldoffset = offsetof(CPUARMState, cp15.sder) },
3891     { .name = "SDER",
3892       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
3893       .access = PL3_RW, .resetvalue = 0,
3894       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
3895     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
3896       .access = PL1_RW, .accessfn = access_trap_aa32s_el1,
3897       .writefn = vbar_write, .resetvalue = 0,
3898       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
3899     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
3900       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
3901       .access = PL3_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
3902       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
3903     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
3904       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
3905       .access = PL3_RW,
3906       /* no .writefn needed as this can't cause an ASID change;
3907        * we must provide a .raw_writefn and .resetfn because we handle
3908        * reset and migration for the AArch32 TTBCR(S), which might be
3909        * using mask and base_mask.
3910        */
3911       .resetfn = vmsa_ttbcr_reset, .raw_writefn = vmsa_ttbcr_raw_write,
3912       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
3913     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
3914       .type = ARM_CP_ALIAS,
3915       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
3916       .access = PL3_RW,
3917       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
3918     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
3919       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
3920       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
3921     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
3922       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
3923       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
3924     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
3925       .type = ARM_CP_ALIAS,
3926       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
3927       .access = PL3_RW,
3928       .fieldoffset = offsetof(CPUARMState, banked_spsr[BANK_MON]) },
3929     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
3930       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
3931       .access = PL3_RW, .writefn = vbar_write,
3932       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
3933       .resetvalue = 0 },
3934     { .name = "CPTR_EL3", .state = ARM_CP_STATE_AA64,
3935       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 2,
3936       .access = PL3_RW, .accessfn = cptr_access, .resetvalue = 0,
3937       .fieldoffset = offsetof(CPUARMState, cp15.cptr_el[3]) },
3938     { .name = "TPIDR_EL3", .state = ARM_CP_STATE_AA64,
3939       .opc0 = 3, .opc1 = 6, .crn = 13, .crm = 0, .opc2 = 2,
3940       .access = PL3_RW, .resetvalue = 0,
3941       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[3]) },
3942     { .name = "AMAIR_EL3", .state = ARM_CP_STATE_AA64,
3943       .opc0 = 3, .opc1 = 6, .crn = 10, .crm = 3, .opc2 = 0,
3944       .access = PL3_RW, .type = ARM_CP_CONST,
3945       .resetvalue = 0 },
3946     { .name = "AFSR0_EL3", .state = ARM_CP_STATE_BOTH,
3947       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 0,
3948       .access = PL3_RW, .type = ARM_CP_CONST,
3949       .resetvalue = 0 },
3950     { .name = "AFSR1_EL3", .state = ARM_CP_STATE_BOTH,
3951       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 1, .opc2 = 1,
3952       .access = PL3_RW, .type = ARM_CP_CONST,
3953       .resetvalue = 0 },
3954     { .name = "TLBI_ALLE3IS", .state = ARM_CP_STATE_AA64,
3955       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 0,
3956       .access = PL3_W, .type = ARM_CP_NO_RAW,
3957       .writefn = tlbi_aa64_alle3is_write },
3958     { .name = "TLBI_VAE3IS", .state = ARM_CP_STATE_AA64,
3959       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 1,
3960       .access = PL3_W, .type = ARM_CP_NO_RAW,
3961       .writefn = tlbi_aa64_vae3is_write },
3962     { .name = "TLBI_VALE3IS", .state = ARM_CP_STATE_AA64,
3963       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 3, .opc2 = 5,
3964       .access = PL3_W, .type = ARM_CP_NO_RAW,
3965       .writefn = tlbi_aa64_vae3is_write },
3966     { .name = "TLBI_ALLE3", .state = ARM_CP_STATE_AA64,
3967       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 0,
3968       .access = PL3_W, .type = ARM_CP_NO_RAW,
3969       .writefn = tlbi_aa64_alle3_write },
3970     { .name = "TLBI_VAE3", .state = ARM_CP_STATE_AA64,
3971       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 1,
3972       .access = PL3_W, .type = ARM_CP_NO_RAW,
3973       .writefn = tlbi_aa64_vae3_write },
3974     { .name = "TLBI_VALE3", .state = ARM_CP_STATE_AA64,
3975       .opc0 = 1, .opc1 = 6, .crn = 8, .crm = 7, .opc2 = 5,
3976       .access = PL3_W, .type = ARM_CP_NO_RAW,
3977       .writefn = tlbi_aa64_vae3_write },
3978     REGINFO_SENTINEL
3979 };
3980
3981 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri,
3982                                      bool isread)
3983 {
3984     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
3985      * but the AArch32 CTR has its own reginfo struct)
3986      */
3987     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
3988         return CP_ACCESS_TRAP;
3989     }
3990     return CP_ACCESS_OK;
3991 }
3992
3993 static void oslar_write(CPUARMState *env, const ARMCPRegInfo *ri,
3994                         uint64_t value)
3995 {
3996     /* Writes to OSLAR_EL1 may update the OS lock status, which can be
3997      * read via a bit in OSLSR_EL1.
3998      */
3999     int oslock;
4000
4001     if (ri->state == ARM_CP_STATE_AA32) {
4002         oslock = (value == 0xC5ACCE55);
4003     } else {
4004         oslock = value & 1;
4005     }
4006
4007     env->cp15.oslsr_el1 = deposit32(env->cp15.oslsr_el1, 1, 1, oslock);
4008 }
4009
4010 static const ARMCPRegInfo debug_cp_reginfo[] = {
4011     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
4012      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
4013      * unlike DBGDRAR it is never accessible from EL0.
4014      * DBGDSAR is deprecated and must RAZ from v8 anyway, so it has no AArch64
4015      * accessor.
4016      */
4017     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
4018       .access = PL0_R, .accessfn = access_tdra,
4019       .type = ARM_CP_CONST, .resetvalue = 0 },
4020     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
4021       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
4022       .access = PL1_R, .accessfn = access_tdra,
4023       .type = ARM_CP_CONST, .resetvalue = 0 },
4024     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
4025       .access = PL0_R, .accessfn = access_tdra,
4026       .type = ARM_CP_CONST, .resetvalue = 0 },
4027     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
4028     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
4029       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
4030       .access = PL1_RW, .accessfn = access_tda,
4031       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
4032       .resetvalue = 0 },
4033     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
4034      * We don't implement the configurable EL0 access.
4035      */
4036     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
4037       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
4038       .type = ARM_CP_ALIAS,
4039       .access = PL1_R, .accessfn = access_tda,
4040       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1), },
4041     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
4042       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
4043       .access = PL1_W, .type = ARM_CP_NO_RAW,
4044       .accessfn = access_tdosa,
4045       .writefn = oslar_write },
4046     { .name = "OSLSR_EL1", .state = ARM_CP_STATE_BOTH,
4047       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 4,
4048       .access = PL1_R, .resetvalue = 10,
4049       .accessfn = access_tdosa,
4050       .fieldoffset = offsetof(CPUARMState, cp15.oslsr_el1) },
4051     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
4052     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
4053       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
4054       .access = PL1_RW, .accessfn = access_tdosa,
4055       .type = ARM_CP_NOP },
4056     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
4057      * implement vector catch debug events yet.
4058      */
4059     { .name = "DBGVCR",
4060       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
4061       .access = PL1_RW, .accessfn = access_tda,
4062       .type = ARM_CP_NOP },
4063     REGINFO_SENTINEL
4064 };
4065
4066 static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
4067     /* 64 bit access versions of the (dummy) debug registers */
4068     { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
4069       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
4070     { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
4071       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
4072     REGINFO_SENTINEL
4073 };
4074
4075 void hw_watchpoint_update(ARMCPU *cpu, int n)
4076 {
4077     CPUARMState *env = &cpu->env;
4078     vaddr len = 0;
4079     vaddr wvr = env->cp15.dbgwvr[n];
4080     uint64_t wcr = env->cp15.dbgwcr[n];
4081     int mask;
4082     int flags = BP_CPU | BP_STOP_BEFORE_ACCESS;
4083
4084     if (env->cpu_watchpoint[n]) {
4085         cpu_watchpoint_remove_by_ref(CPU(cpu), env->cpu_watchpoint[n]);
4086         env->cpu_watchpoint[n] = NULL;
4087     }
4088
4089     if (!extract64(wcr, 0, 1)) {
4090         /* E bit clear : watchpoint disabled */
4091         return;
4092     }
4093
4094     switch (extract64(wcr, 3, 2)) {
4095     case 0:
4096         /* LSC 00 is reserved and must behave as if the wp is disabled */
4097         return;
4098     case 1:
4099         flags |= BP_MEM_READ;
4100         break;
4101     case 2:
4102         flags |= BP_MEM_WRITE;
4103         break;
4104     case 3:
4105         flags |= BP_MEM_ACCESS;
4106         break;
4107     }
4108
4109     /* Attempts to use both MASK and BAS fields simultaneously are
4110      * CONSTRAINED UNPREDICTABLE; we opt to ignore BAS in this case,
4111      * thus generating a watchpoint for every byte in the masked region.
4112      */
4113     mask = extract64(wcr, 24, 4);
4114     if (mask == 1 || mask == 2) {
4115         /* Reserved values of MASK; we must act as if the mask value was
4116          * some non-reserved value, or as if the watchpoint were disabled.
4117          * We choose the latter.
4118          */
4119         return;
4120     } else if (mask) {
4121         /* Watchpoint covers an aligned area up to 2GB in size */
4122         len = 1ULL << mask;
4123         /* If masked bits in WVR are not zero it's CONSTRAINED UNPREDICTABLE
4124          * whether the watchpoint fires when the unmasked bits match; we opt
4125          * to generate the exceptions.
4126          */
4127         wvr &= ~(len - 1);
4128     } else {
4129         /* Watchpoint covers bytes defined by the byte address select bits */
4130         int bas = extract64(wcr, 5, 8);
4131         int basstart;
4132
4133         if (bas == 0) {
4134             /* This must act as if the watchpoint is disabled */
4135             return;
4136         }
4137
4138         if (extract64(wvr, 2, 1)) {
4139             /* Deprecated case of an only 4-aligned address. BAS[7:4] are
4140              * ignored, and BAS[3:0] define which bytes to watch.
4141              */
4142             bas &= 0xf;
4143         }
4144         /* The BAS bits are supposed to be programmed to indicate a contiguous
4145          * range of bytes. Otherwise it is CONSTRAINED UNPREDICTABLE whether
4146          * we fire for each byte in the word/doubleword addressed by the WVR.
4147          * We choose to ignore any non-zero bits after the first range of 1s.
4148          */
4149         basstart = ctz32(bas);
4150         len = cto32(bas >> basstart);
4151         wvr += basstart;
4152     }
4153
4154     cpu_watchpoint_insert(CPU(cpu), wvr, len, flags,
4155                           &env->cpu_watchpoint[n]);
4156 }
4157
4158 void hw_watchpoint_update_all(ARMCPU *cpu)
4159 {
4160     int i;
4161     CPUARMState *env = &cpu->env;
4162
4163     /* Completely clear out existing QEMU watchpoints and our array, to
4164      * avoid possible stale entries following migration load.
4165      */
4166     cpu_watchpoint_remove_all(CPU(cpu), BP_CPU);
4167     memset(env->cpu_watchpoint, 0, sizeof(env->cpu_watchpoint));
4168
4169     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_watchpoint); i++) {
4170         hw_watchpoint_update(cpu, i);
4171     }
4172 }
4173
4174 static void dbgwvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4175                          uint64_t value)
4176 {
4177     ARMCPU *cpu = arm_env_get_cpu(env);
4178     int i = ri->crm;
4179
4180     /* Bits [63:49] are hardwired to the value of bit [48]; that is, the
4181      * register reads and behaves as if values written are sign extended.
4182      * Bits [1:0] are RES0.
4183      */
4184     value = sextract64(value, 0, 49) & ~3ULL;
4185
4186     raw_write(env, ri, value);
4187     hw_watchpoint_update(cpu, i);
4188 }
4189
4190 static void dbgwcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4191                          uint64_t value)
4192 {
4193     ARMCPU *cpu = arm_env_get_cpu(env);
4194     int i = ri->crm;
4195
4196     raw_write(env, ri, value);
4197     hw_watchpoint_update(cpu, i);
4198 }
4199
4200 void hw_breakpoint_update(ARMCPU *cpu, int n)
4201 {
4202     CPUARMState *env = &cpu->env;
4203     uint64_t bvr = env->cp15.dbgbvr[n];
4204     uint64_t bcr = env->cp15.dbgbcr[n];
4205     vaddr addr;
4206     int bt;
4207     int flags = BP_CPU;
4208
4209     if (env->cpu_breakpoint[n]) {
4210         cpu_breakpoint_remove_by_ref(CPU(cpu), env->cpu_breakpoint[n]);
4211         env->cpu_breakpoint[n] = NULL;
4212     }
4213
4214     if (!extract64(bcr, 0, 1)) {
4215         /* E bit clear : watchpoint disabled */
4216         return;
4217     }
4218
4219     bt = extract64(bcr, 20, 4);
4220
4221     switch (bt) {
4222     case 4: /* unlinked address mismatch (reserved if AArch64) */
4223     case 5: /* linked address mismatch (reserved if AArch64) */
4224         qemu_log_mask(LOG_UNIMP,
4225                       "arm: address mismatch breakpoint types not implemented");
4226         return;
4227     case 0: /* unlinked address match */
4228     case 1: /* linked address match */
4229     {
4230         /* Bits [63:49] are hardwired to the value of bit [48]; that is,
4231          * we behave as if the register was sign extended. Bits [1:0] are
4232          * RES0. The BAS field is used to allow setting breakpoints on 16
4233          * bit wide instructions; it is CONSTRAINED UNPREDICTABLE whether
4234          * a bp will fire if the addresses covered by the bp and the addresses
4235          * covered by the insn overlap but the insn doesn't start at the
4236          * start of the bp address range. We choose to require the insn and
4237          * the bp to have the same address. The constraints on writing to
4238          * BAS enforced in dbgbcr_write mean we have only four cases:
4239          *  0b0000  => no breakpoint
4240          *  0b0011  => breakpoint on addr
4241          *  0b1100  => breakpoint on addr + 2
4242          *  0b1111  => breakpoint on addr
4243          * See also figure D2-3 in the v8 ARM ARM (DDI0487A.c).
4244          */
4245         int bas = extract64(bcr, 5, 4);
4246         addr = sextract64(bvr, 0, 49) & ~3ULL;
4247         if (bas == 0) {
4248             return;
4249         }
4250         if (bas == 0xc) {
4251             addr += 2;
4252         }
4253         break;
4254     }
4255     case 2: /* unlinked context ID match */
4256     case 8: /* unlinked VMID match (reserved if no EL2) */
4257     case 10: /* unlinked context ID and VMID match (reserved if no EL2) */
4258         qemu_log_mask(LOG_UNIMP,
4259                       "arm: unlinked context breakpoint types not implemented");
4260         return;
4261     case 9: /* linked VMID match (reserved if no EL2) */
4262     case 11: /* linked context ID and VMID match (reserved if no EL2) */
4263     case 3: /* linked context ID match */
4264     default:
4265         /* We must generate no events for Linked context matches (unless
4266          * they are linked to by some other bp/wp, which is handled in
4267          * updates for the linking bp/wp). We choose to also generate no events
4268          * for reserved values.
4269          */
4270         return;
4271     }
4272
4273     cpu_breakpoint_insert(CPU(cpu), addr, flags, &env->cpu_breakpoint[n]);
4274 }
4275
4276 void hw_breakpoint_update_all(ARMCPU *cpu)
4277 {
4278     int i;
4279     CPUARMState *env = &cpu->env;
4280
4281     /* Completely clear out existing QEMU breakpoints and our array, to
4282      * avoid possible stale entries following migration load.
4283      */
4284     cpu_breakpoint_remove_all(CPU(cpu), BP_CPU);
4285     memset(env->cpu_breakpoint, 0, sizeof(env->cpu_breakpoint));
4286
4287     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_breakpoint); i++) {
4288         hw_breakpoint_update(cpu, i);
4289     }
4290 }
4291
4292 static void dbgbvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4293                          uint64_t value)
4294 {
4295     ARMCPU *cpu = arm_env_get_cpu(env);
4296     int i = ri->crm;
4297
4298     raw_write(env, ri, value);
4299     hw_breakpoint_update(cpu, i);
4300 }
4301
4302 static void dbgbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
4303                          uint64_t value)
4304 {
4305     ARMCPU *cpu = arm_env_get_cpu(env);
4306     int i = ri->crm;
4307
4308     /* BAS[3] is a read-only copy of BAS[2], and BAS[1] a read-only
4309      * copy of BAS[0].
4310      */
4311     value = deposit64(value, 6, 1, extract64(value, 5, 1));
4312     value = deposit64(value, 8, 1, extract64(value, 7, 1));
4313
4314     raw_write(env, ri, value);
4315     hw_breakpoint_update(cpu, i);
4316 }
4317
4318 static void define_debug_regs(ARMCPU *cpu)
4319 {
4320     /* Define v7 and v8 architectural debug registers.
4321      * These are just dummy implementations for now.
4322      */
4323     int i;
4324     int wrps, brps, ctx_cmps;
4325     ARMCPRegInfo dbgdidr = {
4326         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
4327         .access = PL0_R, .accessfn = access_tda,
4328         .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
4329     };
4330
4331     /* Note that all these register fields hold "number of Xs minus 1". */
4332     brps = extract32(cpu->dbgdidr, 24, 4);
4333     wrps = extract32(cpu->dbgdidr, 28, 4);
4334     ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
4335
4336     assert(ctx_cmps <= brps);
4337
4338     /* The DBGDIDR and ID_AA64DFR0_EL1 define various properties
4339      * of the debug registers such as number of breakpoints;
4340      * check that if they both exist then they agree.
4341      */
4342     if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
4343         assert(extract32(cpu->id_aa64dfr0, 12, 4) == brps);
4344         assert(extract32(cpu->id_aa64dfr0, 20, 4) == wrps);
4345         assert(extract32(cpu->id_aa64dfr0, 28, 4) == ctx_cmps);
4346     }
4347
4348     define_one_arm_cp_reg(cpu, &dbgdidr);
4349     define_arm_cp_regs(cpu, debug_cp_reginfo);
4350
4351     if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
4352         define_arm_cp_regs(cpu, debug_lpae_cp_reginfo);
4353     }
4354
4355     for (i = 0; i < brps + 1; i++) {
4356         ARMCPRegInfo dbgregs[] = {
4357             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
4358               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
4359               .access = PL1_RW, .accessfn = access_tda,
4360               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
4361               .writefn = dbgbvr_write, .raw_writefn = raw_write
4362             },
4363             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
4364               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
4365               .access = PL1_RW, .accessfn = access_tda,
4366               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
4367               .writefn = dbgbcr_write, .raw_writefn = raw_write
4368             },
4369             REGINFO_SENTINEL
4370         };
4371         define_arm_cp_regs(cpu, dbgregs);
4372     }
4373
4374     for (i = 0; i < wrps + 1; i++) {
4375         ARMCPRegInfo dbgregs[] = {
4376             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
4377               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
4378               .access = PL1_RW, .accessfn = access_tda,
4379               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
4380               .writefn = dbgwvr_write, .raw_writefn = raw_write
4381             },
4382             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
4383               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
4384               .access = PL1_RW, .accessfn = access_tda,
4385               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
4386               .writefn = dbgwcr_write, .raw_writefn = raw_write
4387             },
4388             REGINFO_SENTINEL
4389         };
4390         define_arm_cp_regs(cpu, dbgregs);
4391     }
4392 }
4393
4394 void register_cp_regs_for_features(ARMCPU *cpu)
4395 {
4396     /* Register all the coprocessor registers based on feature bits */
4397     CPUARMState *env = &cpu->env;
4398     if (arm_feature(env, ARM_FEATURE_M)) {
4399         /* M profile has no coprocessor registers */
4400         return;
4401     }
4402
4403     define_arm_cp_regs(cpu, cp_reginfo);
4404     if (!arm_feature(env, ARM_FEATURE_V8)) {
4405         /* Must go early as it is full of wildcards that may be
4406          * overridden by later definitions.
4407          */
4408         define_arm_cp_regs(cpu, not_v8_cp_reginfo);
4409     }
4410
4411     if (arm_feature(env, ARM_FEATURE_V6)) {
4412         /* The ID registers all have impdef reset values */
4413         ARMCPRegInfo v6_idregs[] = {
4414             { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
4415               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
4416               .access = PL1_R, .type = ARM_CP_CONST,
4417               .resetvalue = cpu->id_pfr0 },
4418             { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
4419               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
4420               .access = PL1_R, .type = ARM_CP_CONST,
4421               .resetvalue = cpu->id_pfr1 },
4422             { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
4423               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
4424               .access = PL1_R, .type = ARM_CP_CONST,
4425               .resetvalue = cpu->id_dfr0 },
4426             { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
4427               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
4428               .access = PL1_R, .type = ARM_CP_CONST,
4429               .resetvalue = cpu->id_afr0 },
4430             { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
4431               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
4432               .access = PL1_R, .type = ARM_CP_CONST,
4433               .resetvalue = cpu->id_mmfr0 },
4434             { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
4435               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
4436               .access = PL1_R, .type = ARM_CP_CONST,
4437               .resetvalue = cpu->id_mmfr1 },
4438             { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
4439               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
4440               .access = PL1_R, .type = ARM_CP_CONST,
4441               .resetvalue = cpu->id_mmfr2 },
4442             { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
4443               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
4444               .access = PL1_R, .type = ARM_CP_CONST,
4445               .resetvalue = cpu->id_mmfr3 },
4446             { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
4447               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
4448               .access = PL1_R, .type = ARM_CP_CONST,
4449               .resetvalue = cpu->id_isar0 },
4450             { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
4451               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
4452               .access = PL1_R, .type = ARM_CP_CONST,
4453               .resetvalue = cpu->id_isar1 },
4454             { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
4455               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
4456               .access = PL1_R, .type = ARM_CP_CONST,
4457               .resetvalue = cpu->id_isar2 },
4458             { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
4459               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
4460               .access = PL1_R, .type = ARM_CP_CONST,
4461               .resetvalue = cpu->id_isar3 },
4462             { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
4463               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
4464               .access = PL1_R, .type = ARM_CP_CONST,
4465               .resetvalue = cpu->id_isar4 },
4466             { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
4467               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
4468               .access = PL1_R, .type = ARM_CP_CONST,
4469               .resetvalue = cpu->id_isar5 },
4470             { .name = "ID_MMFR4", .state = ARM_CP_STATE_BOTH,
4471               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 6,
4472               .access = PL1_R, .type = ARM_CP_CONST,
4473               .resetvalue = cpu->id_mmfr4 },
4474             /* 7 is as yet unallocated and must RAZ */
4475             { .name = "ID_ISAR7_RESERVED", .state = ARM_CP_STATE_BOTH,
4476               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 7,
4477               .access = PL1_R, .type = ARM_CP_CONST,
4478               .resetvalue = 0 },
4479             REGINFO_SENTINEL
4480         };
4481         define_arm_cp_regs(cpu, v6_idregs);
4482         define_arm_cp_regs(cpu, v6_cp_reginfo);
4483     } else {
4484         define_arm_cp_regs(cpu, not_v6_cp_reginfo);
4485     }
4486     if (arm_feature(env, ARM_FEATURE_V6K)) {
4487         define_arm_cp_regs(cpu, v6k_cp_reginfo);
4488     }
4489     if (arm_feature(env, ARM_FEATURE_V7MP) &&
4490         !arm_feature(env, ARM_FEATURE_MPU)) {
4491         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
4492     }
4493     if (arm_feature(env, ARM_FEATURE_V7)) {
4494         /* v7 performance monitor control register: same implementor
4495          * field as main ID register, and we implement only the cycle
4496          * count register.
4497          */
4498 #ifndef CONFIG_USER_ONLY
4499         ARMCPRegInfo pmcr = {
4500             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
4501             .access = PL0_RW,
4502             .type = ARM_CP_IO | ARM_CP_ALIAS,
4503             .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
4504             .accessfn = pmreg_access, .writefn = pmcr_write,
4505             .raw_writefn = raw_write,
4506         };
4507         ARMCPRegInfo pmcr64 = {
4508             .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
4509             .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
4510             .access = PL0_RW, .accessfn = pmreg_access,
4511             .type = ARM_CP_IO,
4512             .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
4513             .resetvalue = cpu->midr & 0xff000000,
4514             .writefn = pmcr_write, .raw_writefn = raw_write,
4515         };
4516         define_one_arm_cp_reg(cpu, &pmcr);
4517         define_one_arm_cp_reg(cpu, &pmcr64);
4518 #endif
4519         ARMCPRegInfo clidr = {
4520             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
4521             .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
4522             .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->clidr
4523         };
4524         define_one_arm_cp_reg(cpu, &clidr);
4525         define_arm_cp_regs(cpu, v7_cp_reginfo);
4526         define_debug_regs(cpu);
4527     } else {
4528         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
4529     }
4530     if (arm_feature(env, ARM_FEATURE_V8)) {
4531         /* AArch64 ID registers, which all have impdef reset values.
4532          * Note that within the ID register ranges the unused slots
4533          * must all RAZ, not UNDEF; future architecture versions may
4534          * define new registers here.
4535          */
4536         ARMCPRegInfo v8_idregs[] = {
4537             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
4538               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
4539               .access = PL1_R, .type = ARM_CP_CONST,
4540               .resetvalue = cpu->id_aa64pfr0 },
4541             { .name = "ID_AA64PFR1_EL1", .state = ARM_CP_STATE_AA64,
4542               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
4543               .access = PL1_R, .type = ARM_CP_CONST,
4544               .resetvalue = cpu->id_aa64pfr1},
4545             { .name = "ID_AA64PFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4546               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 2,
4547               .access = PL1_R, .type = ARM_CP_CONST,
4548               .resetvalue = 0 },
4549             { .name = "ID_AA64PFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4550               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 3,
4551               .access = PL1_R, .type = ARM_CP_CONST,
4552               .resetvalue = 0 },
4553             { .name = "ID_AA64PFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4554               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 4,
4555               .access = PL1_R, .type = ARM_CP_CONST,
4556               .resetvalue = 0 },
4557             { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4558               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5,
4559               .access = PL1_R, .type = ARM_CP_CONST,
4560               .resetvalue = 0 },
4561             { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4562               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6,
4563               .access = PL1_R, .type = ARM_CP_CONST,
4564               .resetvalue = 0 },
4565             { .name = "ID_AA64PFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4566               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 7,
4567               .access = PL1_R, .type = ARM_CP_CONST,
4568               .resetvalue = 0 },
4569             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
4570               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
4571               .access = PL1_R, .type = ARM_CP_CONST,
4572               /* We mask out the PMUVer field, because we don't currently
4573                * implement the PMU. Not advertising it prevents the guest
4574                * from trying to use it and getting UNDEFs on registers we
4575                * don't implement.
4576                */
4577               .resetvalue = cpu->id_aa64dfr0 & ~0xf00 },
4578             { .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
4579               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
4580               .access = PL1_R, .type = ARM_CP_CONST,
4581               .resetvalue = cpu->id_aa64dfr1 },
4582             { .name = "ID_AA64DFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4583               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 2,
4584               .access = PL1_R, .type = ARM_CP_CONST,
4585               .resetvalue = 0 },
4586             { .name = "ID_AA64DFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4587               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 3,
4588               .access = PL1_R, .type = ARM_CP_CONST,
4589               .resetvalue = 0 },
4590             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
4591               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
4592               .access = PL1_R, .type = ARM_CP_CONST,
4593               .resetvalue = cpu->id_aa64afr0 },
4594             { .name = "ID_AA64AFR1_EL1", .state = ARM_CP_STATE_AA64,
4595               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
4596               .access = PL1_R, .type = ARM_CP_CONST,
4597               .resetvalue = cpu->id_aa64afr1 },
4598             { .name = "ID_AA64AFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4599               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 6,
4600               .access = PL1_R, .type = ARM_CP_CONST,
4601               .resetvalue = 0 },
4602             { .name = "ID_AA64AFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4603               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 7,
4604               .access = PL1_R, .type = ARM_CP_CONST,
4605               .resetvalue = 0 },
4606             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
4607               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
4608               .access = PL1_R, .type = ARM_CP_CONST,
4609               .resetvalue = cpu->id_aa64isar0 },
4610             { .name = "ID_AA64ISAR1_EL1", .state = ARM_CP_STATE_AA64,
4611               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
4612               .access = PL1_R, .type = ARM_CP_CONST,
4613               .resetvalue = cpu->id_aa64isar1 },
4614             { .name = "ID_AA64ISAR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4615               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 2,
4616               .access = PL1_R, .type = ARM_CP_CONST,
4617               .resetvalue = 0 },
4618             { .name = "ID_AA64ISAR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4619               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 3,
4620               .access = PL1_R, .type = ARM_CP_CONST,
4621               .resetvalue = 0 },
4622             { .name = "ID_AA64ISAR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4623               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 4,
4624               .access = PL1_R, .type = ARM_CP_CONST,
4625               .resetvalue = 0 },
4626             { .name = "ID_AA64ISAR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4627               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 5,
4628               .access = PL1_R, .type = ARM_CP_CONST,
4629               .resetvalue = 0 },
4630             { .name = "ID_AA64ISAR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4631               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 6,
4632               .access = PL1_R, .type = ARM_CP_CONST,
4633               .resetvalue = 0 },
4634             { .name = "ID_AA64ISAR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4635               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 7,
4636               .access = PL1_R, .type = ARM_CP_CONST,
4637               .resetvalue = 0 },
4638             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
4639               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
4640               .access = PL1_R, .type = ARM_CP_CONST,
4641               .resetvalue = cpu->id_aa64mmfr0 },
4642             { .name = "ID_AA64MMFR1_EL1", .state = ARM_CP_STATE_AA64,
4643               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
4644               .access = PL1_R, .type = ARM_CP_CONST,
4645               .resetvalue = cpu->id_aa64mmfr1 },
4646             { .name = "ID_AA64MMFR2_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4647               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 2,
4648               .access = PL1_R, .type = ARM_CP_CONST,
4649               .resetvalue = 0 },
4650             { .name = "ID_AA64MMFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4651               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 3,
4652               .access = PL1_R, .type = ARM_CP_CONST,
4653               .resetvalue = 0 },
4654             { .name = "ID_AA64MMFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4655               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 4,
4656               .access = PL1_R, .type = ARM_CP_CONST,
4657               .resetvalue = 0 },
4658             { .name = "ID_AA64MMFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4659               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 5,
4660               .access = PL1_R, .type = ARM_CP_CONST,
4661               .resetvalue = 0 },
4662             { .name = "ID_AA64MMFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4663               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 6,
4664               .access = PL1_R, .type = ARM_CP_CONST,
4665               .resetvalue = 0 },
4666             { .name = "ID_AA64MMFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4667               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 7,
4668               .access = PL1_R, .type = ARM_CP_CONST,
4669               .resetvalue = 0 },
4670             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
4671               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
4672               .access = PL1_R, .type = ARM_CP_CONST,
4673               .resetvalue = cpu->mvfr0 },
4674             { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
4675               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
4676               .access = PL1_R, .type = ARM_CP_CONST,
4677               .resetvalue = cpu->mvfr1 },
4678             { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
4679               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
4680               .access = PL1_R, .type = ARM_CP_CONST,
4681               .resetvalue = cpu->mvfr2 },
4682             { .name = "MVFR3_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4683               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 3,
4684               .access = PL1_R, .type = ARM_CP_CONST,
4685               .resetvalue = 0 },
4686             { .name = "MVFR4_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4687               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 4,
4688               .access = PL1_R, .type = ARM_CP_CONST,
4689               .resetvalue = 0 },
4690             { .name = "MVFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4691               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 5,
4692               .access = PL1_R, .type = ARM_CP_CONST,
4693               .resetvalue = 0 },
4694             { .name = "MVFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4695               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 6,
4696               .access = PL1_R, .type = ARM_CP_CONST,
4697               .resetvalue = 0 },
4698             { .name = "MVFR7_EL1_RESERVED", .state = ARM_CP_STATE_AA64,
4699               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 7,
4700               .access = PL1_R, .type = ARM_CP_CONST,
4701               .resetvalue = 0 },
4702             { .name = "PMCEID0", .state = ARM_CP_STATE_AA32,
4703               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 6,
4704               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4705               .resetvalue = cpu->pmceid0 },
4706             { .name = "PMCEID0_EL0", .state = ARM_CP_STATE_AA64,
4707               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 6,
4708               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4709               .resetvalue = cpu->pmceid0 },
4710             { .name = "PMCEID1", .state = ARM_CP_STATE_AA32,
4711               .cp = 15, .opc1 = 0, .crn = 9, .crm = 12, .opc2 = 7,
4712               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4713               .resetvalue = cpu->pmceid1 },
4714             { .name = "PMCEID1_EL0", .state = ARM_CP_STATE_AA64,
4715               .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 7,
4716               .access = PL0_R, .accessfn = pmreg_access, .type = ARM_CP_CONST,
4717               .resetvalue = cpu->pmceid1 },
4718             REGINFO_SENTINEL
4719         };
4720         /* RVBAR_EL1 is only implemented if EL1 is the highest EL */
4721         if (!arm_feature(env, ARM_FEATURE_EL3) &&
4722             !arm_feature(env, ARM_FEATURE_EL2)) {
4723             ARMCPRegInfo rvbar = {
4724                 .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
4725                 .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
4726                 .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
4727             };
4728             define_one_arm_cp_reg(cpu, &rvbar);
4729         }
4730         define_arm_cp_regs(cpu, v8_idregs);
4731         define_arm_cp_regs(cpu, v8_cp_reginfo);
4732     }
4733     if (arm_feature(env, ARM_FEATURE_EL2)) {
4734         uint64_t vmpidr_def = mpidr_read_val(env);
4735         ARMCPRegInfo vpidr_regs[] = {
4736             { .name = "VPIDR", .state = ARM_CP_STATE_AA32,
4737               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
4738               .access = PL2_RW, .accessfn = access_el3_aa32ns,
4739               .resetvalue = cpu->midr,
4740               .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
4741             { .name = "VPIDR_EL2", .state = ARM_CP_STATE_AA64,
4742               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
4743               .access = PL2_RW, .resetvalue = cpu->midr,
4744               .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
4745             { .name = "VMPIDR", .state = ARM_CP_STATE_AA32,
4746               .cp = 15, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
4747               .access = PL2_RW, .accessfn = access_el3_aa32ns,
4748               .resetvalue = vmpidr_def,
4749               .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
4750             { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_AA64,
4751               .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
4752               .access = PL2_RW,
4753               .resetvalue = vmpidr_def,
4754               .fieldoffset = offsetof(CPUARMState, cp15.vmpidr_el2) },
4755             REGINFO_SENTINEL
4756         };
4757         define_arm_cp_regs(cpu, vpidr_regs);
4758         define_arm_cp_regs(cpu, el2_cp_reginfo);
4759         /* RVBAR_EL2 is only implemented if EL2 is the highest EL */
4760         if (!arm_feature(env, ARM_FEATURE_EL3)) {
4761             ARMCPRegInfo rvbar = {
4762                 .name = "RVBAR_EL2", .state = ARM_CP_STATE_AA64,
4763                 .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 1,
4764                 .type = ARM_CP_CONST, .access = PL2_R, .resetvalue = cpu->rvbar
4765             };
4766             define_one_arm_cp_reg(cpu, &rvbar);
4767         }
4768     } else {
4769         /* If EL2 is missing but higher ELs are enabled, we need to
4770          * register the no_el2 reginfos.
4771          */
4772         if (arm_feature(env, ARM_FEATURE_EL3)) {
4773             /* When EL3 exists but not EL2, VPIDR and VMPIDR take the value
4774              * of MIDR_EL1 and MPIDR_EL1.
4775              */
4776             ARMCPRegInfo vpidr_regs[] = {
4777                 { .name = "VPIDR_EL2", .state = ARM_CP_STATE_BOTH,
4778                   .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 0,
4779                   .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
4780                   .type = ARM_CP_CONST, .resetvalue = cpu->midr,
4781                   .fieldoffset = offsetof(CPUARMState, cp15.vpidr_el2) },
4782                 { .name = "VMPIDR_EL2", .state = ARM_CP_STATE_BOTH,
4783                   .opc0 = 3, .opc1 = 4, .crn = 0, .crm = 0, .opc2 = 5,
4784                   .access = PL2_RW, .accessfn = access_el3_aa32ns_aa64any,
4785                   .type = ARM_CP_NO_RAW,
4786                   .writefn = arm_cp_write_ignore, .readfn = mpidr_read },
4787                 REGINFO_SENTINEL
4788             };
4789             define_arm_cp_regs(cpu, vpidr_regs);
4790             define_arm_cp_regs(cpu, el3_no_el2_cp_reginfo);
4791         }
4792     }
4793     if (arm_feature(env, ARM_FEATURE_EL3)) {
4794         define_arm_cp_regs(cpu, el3_cp_reginfo);
4795         ARMCPRegInfo el3_regs[] = {
4796             { .name = "RVBAR_EL3", .state = ARM_CP_STATE_AA64,
4797               .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 1,
4798               .type = ARM_CP_CONST, .access = PL3_R, .resetvalue = cpu->rvbar },
4799             { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
4800               .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
4801               .access = PL3_RW,
4802               .raw_writefn = raw_write, .writefn = sctlr_write,
4803               .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]),
4804               .resetvalue = cpu->reset_sctlr },
4805             REGINFO_SENTINEL
4806         };
4807
4808         define_arm_cp_regs(cpu, el3_regs);
4809     }
4810     /* The behaviour of NSACR is sufficiently various that we don't
4811      * try to describe it in a single reginfo:
4812      *  if EL3 is 64 bit, then trap to EL3 from S EL1,
4813      *     reads as constant 0xc00 from NS EL1 and NS EL2
4814      *  if EL3 is 32 bit, then RW at EL3, RO at NS EL1 and NS EL2
4815      *  if v7 without EL3, register doesn't exist
4816      *  if v8 without EL3, reads as constant 0xc00 from NS EL1 and NS EL2
4817      */
4818     if (arm_feature(env, ARM_FEATURE_EL3)) {
4819         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
4820             ARMCPRegInfo nsacr = {
4821                 .name = "NSACR", .type = ARM_CP_CONST,
4822                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
4823                 .access = PL1_RW, .accessfn = nsacr_access,
4824                 .resetvalue = 0xc00
4825             };
4826             define_one_arm_cp_reg(cpu, &nsacr);
4827         } else {
4828             ARMCPRegInfo nsacr = {
4829                 .name = "NSACR",
4830                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
4831                 .access = PL3_RW | PL1_R,
4832                 .resetvalue = 0,
4833                 .fieldoffset = offsetof(CPUARMState, cp15.nsacr)
4834             };
4835             define_one_arm_cp_reg(cpu, &nsacr);
4836         }
4837     } else {
4838         if (arm_feature(env, ARM_FEATURE_V8)) {
4839             ARMCPRegInfo nsacr = {
4840                 .name = "NSACR", .type = ARM_CP_CONST,
4841                 .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
4842                 .access = PL1_R,
4843                 .resetvalue = 0xc00
4844             };
4845             define_one_arm_cp_reg(cpu, &nsacr);
4846         }
4847     }
4848
4849     if (arm_feature(env, ARM_FEATURE_MPU)) {
4850         if (arm_feature(env, ARM_FEATURE_V6)) {
4851             /* PMSAv6 not implemented */
4852             assert(arm_feature(env, ARM_FEATURE_V7));
4853             define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
4854             define_arm_cp_regs(cpu, pmsav7_cp_reginfo);
4855         } else {
4856             define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
4857         }
4858     } else {
4859         define_arm_cp_regs(cpu, vmsa_pmsa_cp_reginfo);
4860         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
4861     }
4862     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
4863         define_arm_cp_regs(cpu, t2ee_cp_reginfo);
4864     }
4865     if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
4866         define_arm_cp_regs(cpu, generic_timer_cp_reginfo);
4867     }
4868     if (arm_feature(env, ARM_FEATURE_VAPA)) {
4869         define_arm_cp_regs(cpu, vapa_cp_reginfo);
4870     }
4871     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
4872         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
4873     }
4874     if (arm_feature(env, ARM_FEATURE_CACHE_DIRTY_REG)) {
4875         define_arm_cp_regs(cpu, cache_dirty_status_cp_reginfo);
4876     }
4877     if (arm_feature(env, ARM_FEATURE_CACHE_BLOCK_OPS)) {
4878         define_arm_cp_regs(cpu, cache_block_ops_cp_reginfo);
4879     }
4880     if (arm_feature(env, ARM_FEATURE_OMAPCP)) {
4881         define_arm_cp_regs(cpu, omap_cp_reginfo);
4882     }
4883     if (arm_feature(env, ARM_FEATURE_STRONGARM)) {
4884         define_arm_cp_regs(cpu, strongarm_cp_reginfo);
4885     }
4886     if (arm_feature(env, ARM_FEATURE_XSCALE)) {
4887         define_arm_cp_regs(cpu, xscale_cp_reginfo);
4888     }
4889     if (arm_feature(env, ARM_FEATURE_DUMMY_C15_REGS)) {
4890         define_arm_cp_regs(cpu, dummy_c15_cp_reginfo);
4891     }
4892     if (arm_feature(env, ARM_FEATURE_LPAE)) {
4893         define_arm_cp_regs(cpu, lpae_cp_reginfo);
4894     }
4895     /* Slightly awkwardly, the OMAP and StrongARM cores need all of
4896      * cp15 crn=0 to be writes-ignored, whereas for other cores they should
4897      * be read-only (ie write causes UNDEF exception).
4898      */
4899     {
4900         ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
4901             /* Pre-v8 MIDR space.
4902              * Note that the MIDR isn't a simple constant register because
4903              * of the TI925 behaviour where writes to another register can
4904              * cause the MIDR value to change.
4905              *
4906              * Unimplemented registers in the c15 0 0 0 space default to
4907              * MIDR. Define MIDR first as this entire space, then CTR, TCMTR
4908              * and friends override accordingly.
4909              */
4910             { .name = "MIDR",
4911               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
4912               .access = PL1_R, .resetvalue = cpu->midr,
4913               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
4914               .readfn = midr_read,
4915               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
4916               .type = ARM_CP_OVERRIDE },
4917             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
4918             { .name = "DUMMY",
4919               .cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
4920               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4921             { .name = "DUMMY",
4922               .cp = 15, .crn = 0, .crm = 4, .opc1 = 0, .opc2 = CP_ANY,
4923               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4924             { .name = "DUMMY",
4925               .cp = 15, .crn = 0, .crm = 5, .opc1 = 0, .opc2 = CP_ANY,
4926               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4927             { .name = "DUMMY",
4928               .cp = 15, .crn = 0, .crm = 6, .opc1 = 0, .opc2 = CP_ANY,
4929               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4930             { .name = "DUMMY",
4931               .cp = 15, .crn = 0, .crm = 7, .opc1 = 0, .opc2 = CP_ANY,
4932               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4933             REGINFO_SENTINEL
4934         };
4935         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
4936             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
4937               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
4938               .access = PL1_R, .type = ARM_CP_NO_RAW, .resetvalue = cpu->midr,
4939               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
4940               .readfn = midr_read },
4941             /* crn = 0 op1 = 0 crm = 0 op2 = 4,7 : AArch32 aliases of MIDR */
4942             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
4943               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
4944               .access = PL1_R, .resetvalue = cpu->midr },
4945             { .name = "MIDR", .type = ARM_CP_ALIAS | ARM_CP_CONST,
4946               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 7,
4947               .access = PL1_R, .resetvalue = cpu->midr },
4948             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
4949               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
4950               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->revidr },
4951             REGINFO_SENTINEL
4952         };
4953         ARMCPRegInfo id_cp_reginfo[] = {
4954             /* These are common to v8 and pre-v8 */
4955             { .name = "CTR",
4956               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
4957               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
4958             { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
4959               .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
4960               .access = PL0_R, .accessfn = ctr_el0_access,
4961               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
4962             /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
4963             { .name = "TCMTR",
4964               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
4965               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
4966             REGINFO_SENTINEL
4967         };
4968         /* TLBTR is specific to VMSA */
4969         ARMCPRegInfo id_tlbtr_reginfo = {
4970               .name = "TLBTR",
4971               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
4972               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0,
4973         };
4974         /* MPUIR is specific to PMSA V6+ */
4975         ARMCPRegInfo id_mpuir_reginfo = {
4976               .name = "MPUIR",
4977               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 4,
4978               .access = PL1_R, .type = ARM_CP_CONST,
4979               .resetvalue = cpu->pmsav7_dregion << 8
4980         };
4981         ARMCPRegInfo crn0_wi_reginfo = {
4982             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
4983             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
4984             .type = ARM_CP_NOP | ARM_CP_OVERRIDE
4985         };
4986         if (arm_feature(env, ARM_FEATURE_OMAPCP) ||
4987             arm_feature(env, ARM_FEATURE_STRONGARM)) {
4988             ARMCPRegInfo *r;
4989             /* Register the blanket "writes ignored" value first to cover the
4990              * whole space. Then update the specific ID registers to allow write
4991              * access, so that they ignore writes rather than causing them to
4992              * UNDEF.
4993              */
4994             define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
4995             for (r = id_pre_v8_midr_cp_reginfo;
4996                  r->type != ARM_CP_SENTINEL; r++) {
4997                 r->access = PL1_RW;
4998             }
4999             for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
5000                 r->access = PL1_RW;
5001             }
5002             id_tlbtr_reginfo.access = PL1_RW;
5003             id_tlbtr_reginfo.access = PL1_RW;
5004         }
5005         if (arm_feature(env, ARM_FEATURE_V8)) {
5006             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
5007         } else {
5008             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
5009         }
5010         define_arm_cp_regs(cpu, id_cp_reginfo);
5011         if (!arm_feature(env, ARM_FEATURE_MPU)) {
5012             define_one_arm_cp_reg(cpu, &id_tlbtr_reginfo);
5013         } else if (arm_feature(env, ARM_FEATURE_V7)) {
5014             define_one_arm_cp_reg(cpu, &id_mpuir_reginfo);
5015         }
5016     }
5017
5018     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
5019         define_arm_cp_regs(cpu, mpidr_cp_reginfo);
5020     }
5021
5022     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
5023         ARMCPRegInfo auxcr_reginfo[] = {
5024             { .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
5025               .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
5026               .access = PL1_RW, .type = ARM_CP_CONST,
5027               .resetvalue = cpu->reset_auxcr },
5028             { .name = "ACTLR_EL2", .state = ARM_CP_STATE_BOTH,
5029               .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 0, .opc2 = 1,
5030               .access = PL2_RW, .type = ARM_CP_CONST,
5031               .resetvalue = 0 },
5032             { .name = "ACTLR_EL3", .state = ARM_CP_STATE_AA64,
5033               .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 1,
5034               .access = PL3_RW, .type = ARM_CP_CONST,
5035               .resetvalue = 0 },
5036             REGINFO_SENTINEL
5037         };
5038         define_arm_cp_regs(cpu, auxcr_reginfo);
5039     }
5040
5041     if (arm_feature(env, ARM_FEATURE_CBAR)) {
5042         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
5043             /* 32 bit view is [31:18] 0...0 [43:32]. */
5044             uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
5045                 | extract64(cpu->reset_cbar, 32, 12);
5046             ARMCPRegInfo cbar_reginfo[] = {
5047                 { .name = "CBAR",
5048                   .type = ARM_CP_CONST,
5049                   .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
5050                   .access = PL1_R, .resetvalue = cpu->reset_cbar },
5051                 { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
5052                   .type = ARM_CP_CONST,
5053                   .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
5054                   .access = PL1_R, .resetvalue = cbar32 },
5055                 REGINFO_SENTINEL
5056             };
5057             /* We don't implement a r/w 64 bit CBAR currently */
5058             assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
5059             define_arm_cp_regs(cpu, cbar_reginfo);
5060         } else {
5061             ARMCPRegInfo cbar = {
5062                 .name = "CBAR",
5063                 .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
5064                 .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
5065                 .fieldoffset = offsetof(CPUARMState,
5066                                         cp15.c15_config_base_address)
5067             };
5068             if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
5069                 cbar.access = PL1_R;
5070                 cbar.fieldoffset = 0;
5071                 cbar.type = ARM_CP_CONST;
5072             }
5073             define_one_arm_cp_reg(cpu, &cbar);
5074         }
5075     }
5076
5077     /* Generic registers whose values depend on the implementation */
5078     {
5079         ARMCPRegInfo sctlr = {
5080             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
5081             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
5082             .access = PL1_RW,
5083             .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
5084                                    offsetof(CPUARMState, cp15.sctlr_ns) },
5085             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
5086             .raw_writefn = raw_write,
5087         };
5088         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
5089             /* Normally we would always end the TB on an SCTLR write, but Linux
5090              * arch/arm/mach-pxa/sleep.S expects two instructions following
5091              * an MMU enable to execute from cache.  Imitate this behaviour.
5092              */
5093             sctlr.type |= ARM_CP_SUPPRESS_TB_END;
5094         }
5095         define_one_arm_cp_reg(cpu, &sctlr);
5096     }
5097 }
5098
5099 ARMCPU *cpu_arm_init(const char *cpu_model)
5100 {
5101     return ARM_CPU(cpu_generic_init(TYPE_ARM_CPU, cpu_model));
5102 }
5103
5104 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
5105 {
5106     CPUState *cs = CPU(cpu);
5107     CPUARMState *env = &cpu->env;
5108
5109     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
5110         gdb_register_coprocessor(cs, aarch64_fpu_gdb_get_reg,
5111                                  aarch64_fpu_gdb_set_reg,
5112                                  34, "aarch64-fpu.xml", 0);
5113     } else if (arm_feature(env, ARM_FEATURE_NEON)) {
5114         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
5115                                  51, "arm-neon.xml", 0);
5116     } else if (arm_feature(env, ARM_FEATURE_VFP3)) {
5117         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
5118                                  35, "arm-vfp3.xml", 0);
5119     } else if (arm_feature(env, ARM_FEATURE_VFP)) {
5120         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
5121                                  19, "arm-vfp.xml", 0);
5122     }
5123 }
5124
5125 /* Sort alphabetically by type name, except for "any". */
5126 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
5127 {
5128     ObjectClass *class_a = (ObjectClass *)a;
5129     ObjectClass *class_b = (ObjectClass *)b;
5130     const char *name_a, *name_b;
5131
5132     name_a = object_class_get_name(class_a);
5133     name_b = object_class_get_name(class_b);
5134     if (strcmp(name_a, "any-" TYPE_ARM_CPU) == 0) {
5135         return 1;
5136     } else if (strcmp(name_b, "any-" TYPE_ARM_CPU) == 0) {
5137         return -1;
5138     } else {
5139         return strcmp(name_a, name_b);
5140     }
5141 }
5142
5143 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
5144 {
5145     ObjectClass *oc = data;
5146     CPUListState *s = user_data;
5147     const char *typename;
5148     char *name;
5149
5150     typename = object_class_get_name(oc);
5151     name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_ARM_CPU));
5152     (*s->cpu_fprintf)(s->file, "  %s\n",
5153                       name);
5154     g_free(name);
5155 }
5156
5157 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
5158 {
5159     CPUListState s = {
5160         .file = f,
5161         .cpu_fprintf = cpu_fprintf,
5162     };
5163     GSList *list;
5164
5165     list = object_class_get_list(TYPE_ARM_CPU, false);
5166     list = g_slist_sort(list, arm_cpu_list_compare);
5167     (*cpu_fprintf)(f, "Available CPUs:\n");
5168     g_slist_foreach(list, arm_cpu_list_entry, &s);
5169     g_slist_free(list);
5170 #ifdef CONFIG_KVM
5171     /* The 'host' CPU type is dynamically registered only if KVM is
5172      * enabled, so we have to special-case it here:
5173      */
5174     (*cpu_fprintf)(f, "  host (only available in KVM mode)\n");
5175 #endif
5176 }
5177
5178 static void arm_cpu_add_definition(gpointer data, gpointer user_data)
5179 {
5180     ObjectClass *oc = data;
5181     CpuDefinitionInfoList **cpu_list = user_data;
5182     CpuDefinitionInfoList *entry;
5183     CpuDefinitionInfo *info;
5184     const char *typename;
5185
5186     typename = object_class_get_name(oc);
5187     info = g_malloc0(sizeof(*info));
5188     info->name = g_strndup(typename,
5189                            strlen(typename) - strlen("-" TYPE_ARM_CPU));
5190
5191     entry = g_malloc0(sizeof(*entry));
5192     entry->value = info;
5193     entry->next = *cpu_list;
5194     *cpu_list = entry;
5195 }
5196
5197 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
5198 {
5199     CpuDefinitionInfoList *cpu_list = NULL;
5200     GSList *list;
5201
5202     list = object_class_get_list(TYPE_ARM_CPU, false);
5203     g_slist_foreach(list, arm_cpu_add_definition, &cpu_list);
5204     g_slist_free(list);
5205
5206     return cpu_list;
5207 }
5208
5209 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
5210                                    void *opaque, int state, int secstate,
5211                                    int crm, int opc1, int opc2)
5212 {
5213     /* Private utility function for define_one_arm_cp_reg_with_opaque():
5214      * add a single reginfo struct to the hash table.
5215      */
5216     uint32_t *key = g_new(uint32_t, 1);
5217     ARMCPRegInfo *r2 = g_memdup(r, sizeof(ARMCPRegInfo));
5218     int is64 = (r->type & ARM_CP_64BIT) ? 1 : 0;
5219     int ns = (secstate & ARM_CP_SECSTATE_NS) ? 1 : 0;
5220
5221     /* Reset the secure state to the specific incoming state.  This is
5222      * necessary as the register may have been defined with both states.
5223      */
5224     r2->secure = secstate;
5225
5226     if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
5227         /* Register is banked (using both entries in array).
5228          * Overwriting fieldoffset as the array is only used to define
5229          * banked registers but later only fieldoffset is used.
5230          */
5231         r2->fieldoffset = r->bank_fieldoffsets[ns];
5232     }
5233
5234     if (state == ARM_CP_STATE_AA32) {
5235         if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
5236             /* If the register is banked then we don't need to migrate or
5237              * reset the 32-bit instance in certain cases:
5238              *
5239              * 1) If the register has both 32-bit and 64-bit instances then we
5240              *    can count on the 64-bit instance taking care of the
5241              *    non-secure bank.
5242              * 2) If ARMv8 is enabled then we can count on a 64-bit version
5243              *    taking care of the secure bank.  This requires that separate
5244              *    32 and 64-bit definitions are provided.
5245              */
5246             if ((r->state == ARM_CP_STATE_BOTH && ns) ||
5247                 (arm_feature(&cpu->env, ARM_FEATURE_V8) && !ns)) {
5248                 r2->type |= ARM_CP_ALIAS;
5249             }
5250         } else if ((secstate != r->secure) && !ns) {
5251             /* The register is not banked so we only want to allow migration of
5252              * the non-secure instance.
5253              */
5254             r2->type |= ARM_CP_ALIAS;
5255         }
5256
5257         if (r->state == ARM_CP_STATE_BOTH) {
5258             /* We assume it is a cp15 register if the .cp field is left unset.
5259              */
5260             if (r2->cp == 0) {
5261                 r2->cp = 15;
5262             }
5263
5264 #ifdef HOST_WORDS_BIGENDIAN
5265             if (r2->fieldoffset) {
5266                 r2->fieldoffset += sizeof(uint32_t);
5267             }
5268 #endif
5269         }
5270     }
5271     if (state == ARM_CP_STATE_AA64) {
5272         /* To allow abbreviation of ARMCPRegInfo
5273          * definitions, we treat cp == 0 as equivalent to
5274          * the value for "standard guest-visible sysreg".
5275          * STATE_BOTH definitions are also always "standard
5276          * sysreg" in their AArch64 view (the .cp value may
5277          * be non-zero for the benefit of the AArch32 view).
5278          */
5279         if (r->cp == 0 || r->state == ARM_CP_STATE_BOTH) {
5280             r2->cp = CP_REG_ARM64_SYSREG_CP;
5281         }
5282         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
5283                                   r2->opc0, opc1, opc2);
5284     } else {
5285         *key = ENCODE_CP_REG(r2->cp, is64, ns, r2->crn, crm, opc1, opc2);
5286     }
5287     if (opaque) {
5288         r2->opaque = opaque;
5289     }
5290     /* reginfo passed to helpers is correct for the actual access,
5291      * and is never ARM_CP_STATE_BOTH:
5292      */
5293     r2->state = state;
5294     /* Make sure reginfo passed to helpers for wildcarded regs
5295      * has the correct crm/opc1/opc2 for this reg, not CP_ANY:
5296      */
5297     r2->crm = crm;
5298     r2->opc1 = opc1;
5299     r2->opc2 = opc2;
5300     /* By convention, for wildcarded registers only the first
5301      * entry is used for migration; the others are marked as
5302      * ALIAS so we don't try to transfer the register
5303      * multiple times. Special registers (ie NOP/WFI) are
5304      * never migratable and not even raw-accessible.
5305      */
5306     if ((r->type & ARM_CP_SPECIAL)) {
5307         r2->type |= ARM_CP_NO_RAW;
5308     }
5309     if (((r->crm == CP_ANY) && crm != 0) ||
5310         ((r->opc1 == CP_ANY) && opc1 != 0) ||
5311         ((r->opc2 == CP_ANY) && opc2 != 0)) {
5312         r2->type |= ARM_CP_ALIAS;
5313     }
5314
5315     /* Check that raw accesses are either forbidden or handled. Note that
5316      * we can't assert this earlier because the setup of fieldoffset for
5317      * banked registers has to be done first.
5318      */
5319     if (!(r2->type & ARM_CP_NO_RAW)) {
5320         assert(!raw_accessors_invalid(r2));
5321     }
5322
5323     /* Overriding of an existing definition must be explicitly
5324      * requested.
5325      */
5326     if (!(r->type & ARM_CP_OVERRIDE)) {
5327         ARMCPRegInfo *oldreg;
5328         oldreg = g_hash_table_lookup(cpu->cp_regs, key);
5329         if (oldreg && !(oldreg->type & ARM_CP_OVERRIDE)) {
5330             fprintf(stderr, "Register redefined: cp=%d %d bit "
5331                     "crn=%d crm=%d opc1=%d opc2=%d, "
5332                     "was %s, now %s\n", r2->cp, 32 + 32 * is64,
5333                     r2->crn, r2->crm, r2->opc1, r2->opc2,
5334                     oldreg->name, r2->name);
5335             g_assert_not_reached();
5336         }
5337     }
5338     g_hash_table_insert(cpu->cp_regs, key, r2);
5339 }
5340
5341
5342 void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
5343                                        const ARMCPRegInfo *r, void *opaque)
5344 {
5345     /* Define implementations of coprocessor registers.
5346      * We store these in a hashtable because typically
5347      * there are less than 150 registers in a space which
5348      * is 16*16*16*8*8 = 262144 in size.
5349      * Wildcarding is supported for the crm, opc1 and opc2 fields.
5350      * If a register is defined twice then the second definition is
5351      * used, so this can be used to define some generic registers and
5352      * then override them with implementation specific variations.
5353      * At least one of the original and the second definition should
5354      * include ARM_CP_OVERRIDE in its type bits -- this is just a guard
5355      * against accidental use.
5356      *
5357      * The state field defines whether the register is to be
5358      * visible in the AArch32 or AArch64 execution state. If the
5359      * state is set to ARM_CP_STATE_BOTH then we synthesise a
5360      * reginfo structure for the AArch32 view, which sees the lower
5361      * 32 bits of the 64 bit register.
5362      *
5363      * Only registers visible in AArch64 may set r->opc0; opc0 cannot
5364      * be wildcarded. AArch64 registers are always considered to be 64
5365      * bits; the ARM_CP_64BIT* flag applies only to the AArch32 view of
5366      * the register, if any.
5367      */
5368     int crm, opc1, opc2, state;
5369     int crmmin = (r->crm == CP_ANY) ? 0 : r->crm;
5370     int crmmax = (r->crm == CP_ANY) ? 15 : r->crm;
5371     int opc1min = (r->opc1 == CP_ANY) ? 0 : r->opc1;
5372     int opc1max = (r->opc1 == CP_ANY) ? 7 : r->opc1;
5373     int opc2min = (r->opc2 == CP_ANY) ? 0 : r->opc2;
5374     int opc2max = (r->opc2 == CP_ANY) ? 7 : r->opc2;
5375     /* 64 bit registers have only CRm and Opc1 fields */
5376     assert(!((r->type & ARM_CP_64BIT) && (r->opc2 || r->crn)));
5377     /* op0 only exists in the AArch64 encodings */
5378     assert((r->state != ARM_CP_STATE_AA32) || (r->opc0 == 0));
5379     /* AArch64 regs are all 64 bit so ARM_CP_64BIT is meaningless */
5380     assert((r->state != ARM_CP_STATE_AA64) || !(r->type & ARM_CP_64BIT));
5381     /* The AArch64 pseudocode CheckSystemAccess() specifies that op1
5382      * encodes a minimum access level for the register. We roll this
5383      * runtime check into our general permission check code, so check
5384      * here that the reginfo's specified permissions are strict enough
5385      * to encompass the generic architectural permission check.
5386      */
5387     if (r->state != ARM_CP_STATE_AA32) {
5388         int mask = 0;
5389         switch (r->opc1) {
5390         case 0: case 1: case 2:
5391             /* min_EL EL1 */
5392             mask = PL1_RW;
5393             break;
5394         case 3:
5395             /* min_EL EL0 */
5396             mask = PL0_RW;
5397             break;
5398         case 4:
5399             /* min_EL EL2 */
5400             mask = PL2_RW;
5401             break;
5402         case 5:
5403             /* unallocated encoding, so not possible */
5404             assert(false);
5405             break;
5406         case 6:
5407             /* min_EL EL3 */
5408             mask = PL3_RW;
5409             break;
5410         case 7:
5411             /* min_EL EL1, secure mode only (we don't check the latter) */
5412             mask = PL1_RW;
5413             break;
5414         default:
5415             /* broken reginfo with out-of-range opc1 */
5416             assert(false);
5417             break;
5418         }
5419         /* assert our permissions are not too lax (stricter is fine) */
5420         assert((r->access & ~mask) == 0);
5421     }
5422
5423     /* Check that the register definition has enough info to handle
5424      * reads and writes if they are permitted.
5425      */
5426     if (!(r->type & (ARM_CP_SPECIAL|ARM_CP_CONST))) {
5427         if (r->access & PL3_R) {
5428             assert((r->fieldoffset ||
5429                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
5430                    r->readfn);
5431         }
5432         if (r->access & PL3_W) {
5433             assert((r->fieldoffset ||
5434                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
5435                    r->writefn);
5436         }
5437     }
5438     /* Bad type field probably means missing sentinel at end of reg list */
5439     assert(cptype_valid(r->type));
5440     for (crm = crmmin; crm <= crmmax; crm++) {
5441         for (opc1 = opc1min; opc1 <= opc1max; opc1++) {
5442             for (opc2 = opc2min; opc2 <= opc2max; opc2++) {
5443                 for (state = ARM_CP_STATE_AA32;
5444                      state <= ARM_CP_STATE_AA64; state++) {
5445                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
5446                         continue;
5447                     }
5448                     if (state == ARM_CP_STATE_AA32) {
5449                         /* Under AArch32 CP registers can be common
5450                          * (same for secure and non-secure world) or banked.
5451                          */
5452                         switch (r->secure) {
5453                         case ARM_CP_SECSTATE_S:
5454                         case ARM_CP_SECSTATE_NS:
5455                             add_cpreg_to_hashtable(cpu, r, opaque, state,
5456                                                    r->secure, crm, opc1, opc2);
5457                             break;
5458                         default:
5459                             add_cpreg_to_hashtable(cpu, r, opaque, state,
5460                                                    ARM_CP_SECSTATE_S,
5461                                                    crm, opc1, opc2);
5462                             add_cpreg_to_hashtable(cpu, r, opaque, state,
5463                                                    ARM_CP_SECSTATE_NS,
5464                                                    crm, opc1, opc2);
5465                             break;
5466                         }
5467                     } else {
5468                         /* AArch64 registers get mapped to non-secure instance
5469                          * of AArch32 */
5470                         add_cpreg_to_hashtable(cpu, r, opaque, state,
5471                                                ARM_CP_SECSTATE_NS,
5472                                                crm, opc1, opc2);
5473                     }
5474                 }
5475             }
5476         }
5477     }
5478 }
5479
5480 void define_arm_cp_regs_with_opaque(ARMCPU *cpu,
5481                                     const ARMCPRegInfo *regs, void *opaque)
5482 {
5483     /* Define a whole list of registers */
5484     const ARMCPRegInfo *r;
5485     for (r = regs; r->type != ARM_CP_SENTINEL; r++) {
5486         define_one_arm_cp_reg_with_opaque(cpu, r, opaque);
5487     }
5488 }
5489
5490 const ARMCPRegInfo *get_arm_cp_reginfo(GHashTable *cpregs, uint32_t encoded_cp)
5491 {
5492     return g_hash_table_lookup(cpregs, &encoded_cp);
5493 }
5494
5495 void arm_cp_write_ignore(CPUARMState *env, const ARMCPRegInfo *ri,
5496                          uint64_t value)
5497 {
5498     /* Helper coprocessor write function for write-ignore registers */
5499 }
5500
5501 uint64_t arm_cp_read_zero(CPUARMState *env, const ARMCPRegInfo *ri)
5502 {
5503     /* Helper coprocessor write function for read-as-zero registers */
5504     return 0;
5505 }
5506
5507 void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
5508 {
5509     /* Helper coprocessor reset function for do-nothing-on-reset registers */
5510 }
5511
5512 static int bad_mode_switch(CPUARMState *env, int mode, CPSRWriteType write_type)
5513 {
5514     /* Return true if it is not valid for us to switch to
5515      * this CPU mode (ie all the UNPREDICTABLE cases in
5516      * the ARM ARM CPSRWriteByInstr pseudocode).
5517      */
5518
5519     /* Changes to or from Hyp via MSR and CPS are illegal. */
5520     if (write_type == CPSRWriteByInstr &&
5521         ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_HYP ||
5522          mode == ARM_CPU_MODE_HYP)) {
5523         return 1;
5524     }
5525
5526     switch (mode) {
5527     case ARM_CPU_MODE_USR:
5528         return 0;
5529     case ARM_CPU_MODE_SYS:
5530     case ARM_CPU_MODE_SVC:
5531     case ARM_CPU_MODE_ABT:
5532     case ARM_CPU_MODE_UND:
5533     case ARM_CPU_MODE_IRQ:
5534     case ARM_CPU_MODE_FIQ:
5535         /* Note that we don't implement the IMPDEF NSACR.RFR which in v7
5536          * allows FIQ mode to be Secure-only. (In v8 this doesn't exist.)
5537          */
5538         /* If HCR.TGE is set then changes from Monitor to NS PL1 via MSR
5539          * and CPS are treated as illegal mode changes.
5540          */
5541         if (write_type == CPSRWriteByInstr &&
5542             (env->cp15.hcr_el2 & HCR_TGE) &&
5543             (env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON &&
5544             !arm_is_secure_below_el3(env)) {
5545             return 1;
5546         }
5547         return 0;
5548     case ARM_CPU_MODE_HYP:
5549         return !arm_feature(env, ARM_FEATURE_EL2)
5550             || arm_current_el(env) < 2 || arm_is_secure(env);
5551     case ARM_CPU_MODE_MON:
5552         return arm_current_el(env) < 3;
5553     default:
5554         return 1;
5555     }
5556 }
5557
5558 uint32_t cpsr_read(CPUARMState *env)
5559 {
5560     int ZF;
5561     ZF = (env->ZF == 0);
5562     return env->uncached_cpsr | (env->NF & 0x80000000) | (ZF << 30) |
5563         (env->CF << 29) | ((env->VF & 0x80000000) >> 3) | (env->QF << 27)
5564         | (env->thumb << 5) | ((env->condexec_bits & 3) << 25)
5565         | ((env->condexec_bits & 0xfc) << 8)
5566         | (env->GE << 16) | (env->daif & CPSR_AIF);
5567 }
5568
5569 void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask,
5570                 CPSRWriteType write_type)
5571 {
5572     uint32_t changed_daif;
5573
5574     if (mask & CPSR_NZCV) {
5575         env->ZF = (~val) & CPSR_Z;
5576         env->NF = val;
5577         env->CF = (val >> 29) & 1;
5578         env->VF = (val << 3) & 0x80000000;
5579     }
5580     if (mask & CPSR_Q)
5581         env->QF = ((val & CPSR_Q) != 0);
5582     if (mask & CPSR_T)
5583         env->thumb = ((val & CPSR_T) != 0);
5584     if (mask & CPSR_IT_0_1) {
5585         env->condexec_bits &= ~3;
5586         env->condexec_bits |= (val >> 25) & 3;
5587     }
5588     if (mask & CPSR_IT_2_7) {
5589         env->condexec_bits &= 3;
5590         env->condexec_bits |= (val >> 8) & 0xfc;
5591     }
5592     if (mask & CPSR_GE) {
5593         env->GE = (val >> 16) & 0xf;
5594     }
5595
5596     /* In a V7 implementation that includes the security extensions but does
5597      * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
5598      * whether non-secure software is allowed to change the CPSR_F and CPSR_A
5599      * bits respectively.
5600      *
5601      * In a V8 implementation, it is permitted for privileged software to
5602      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
5603      */
5604     if (write_type != CPSRWriteRaw && !arm_feature(env, ARM_FEATURE_V8) &&
5605         arm_feature(env, ARM_FEATURE_EL3) &&
5606         !arm_feature(env, ARM_FEATURE_EL2) &&
5607         !arm_is_secure(env)) {
5608
5609         changed_daif = (env->daif ^ val) & mask;
5610
5611         if (changed_daif & CPSR_A) {
5612             /* Check to see if we are allowed to change the masking of async
5613              * abort exceptions from a non-secure state.
5614              */
5615             if (!(env->cp15.scr_el3 & SCR_AW)) {
5616                 qemu_log_mask(LOG_GUEST_ERROR,
5617                               "Ignoring attempt to switch CPSR_A flag from "
5618                               "non-secure world with SCR.AW bit clear\n");
5619                 mask &= ~CPSR_A;
5620             }
5621         }
5622
5623         if (changed_daif & CPSR_F) {
5624             /* Check to see if we are allowed to change the masking of FIQ
5625              * exceptions from a non-secure state.
5626              */
5627             if (!(env->cp15.scr_el3 & SCR_FW)) {
5628                 qemu_log_mask(LOG_GUEST_ERROR,
5629                               "Ignoring attempt to switch CPSR_F flag from "
5630                               "non-secure world with SCR.FW bit clear\n");
5631                 mask &= ~CPSR_F;
5632             }
5633
5634             /* Check whether non-maskable FIQ (NMFI) support is enabled.
5635              * If this bit is set software is not allowed to mask
5636              * FIQs, but is allowed to set CPSR_F to 0.
5637              */
5638             if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
5639                 (val & CPSR_F)) {
5640                 qemu_log_mask(LOG_GUEST_ERROR,
5641                               "Ignoring attempt to enable CPSR_F flag "
5642                               "(non-maskable FIQ [NMFI] support enabled)\n");
5643                 mask &= ~CPSR_F;
5644             }
5645         }
5646     }
5647
5648     env->daif &= ~(CPSR_AIF & mask);
5649     env->daif |= val & CPSR_AIF & mask;
5650
5651     if (write_type != CPSRWriteRaw &&
5652         ((env->uncached_cpsr ^ val) & mask & CPSR_M)) {
5653         if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR) {
5654             /* Note that we can only get here in USR mode if this is a
5655              * gdb stub write; for this case we follow the architectural
5656              * behaviour for guest writes in USR mode of ignoring an attempt
5657              * to switch mode. (Those are caught by translate.c for writes
5658              * triggered by guest instructions.)
5659              */
5660             mask &= ~CPSR_M;
5661         } else if (bad_mode_switch(env, val & CPSR_M, write_type)) {
5662             /* Attempt to switch to an invalid mode: this is UNPREDICTABLE in
5663              * v7, and has defined behaviour in v8:
5664              *  + leave CPSR.M untouched
5665              *  + allow changes to the other CPSR fields
5666              *  + set PSTATE.IL
5667              * For user changes via the GDB stub, we don't set PSTATE.IL,
5668              * as this would be unnecessarily harsh for a user error.
5669              */
5670             mask &= ~CPSR_M;
5671             if (write_type != CPSRWriteByGDBStub &&
5672                 arm_feature(env, ARM_FEATURE_V8)) {
5673                 mask |= CPSR_IL;
5674                 val |= CPSR_IL;
5675             }
5676         } else {
5677             switch_mode(env, val & CPSR_M);
5678         }
5679     }
5680     mask &= ~CACHED_CPSR_BITS;
5681     env->uncached_cpsr = (env->uncached_cpsr & ~mask) | (val & mask);
5682 }
5683
5684 /* Sign/zero extend */
5685 uint32_t HELPER(sxtb16)(uint32_t x)
5686 {
5687     uint32_t res;
5688     res = (uint16_t)(int8_t)x;
5689     res |= (uint32_t)(int8_t)(x >> 16) << 16;
5690     return res;
5691 }
5692
5693 uint32_t HELPER(uxtb16)(uint32_t x)
5694 {
5695     uint32_t res;
5696     res = (uint16_t)(uint8_t)x;
5697     res |= (uint32_t)(uint8_t)(x >> 16) << 16;
5698     return res;
5699 }
5700
5701 uint32_t HELPER(clz)(uint32_t x)
5702 {
5703     return clz32(x);
5704 }
5705
5706 int32_t HELPER(sdiv)(int32_t num, int32_t den)
5707 {
5708     if (den == 0)
5709       return 0;
5710     if (num == INT_MIN && den == -1)
5711       return INT_MIN;
5712     return num / den;
5713 }
5714
5715 uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
5716 {
5717     if (den == 0)
5718       return 0;
5719     return num / den;
5720 }
5721
5722 uint32_t HELPER(rbit)(uint32_t x)
5723 {
5724     return revbit32(x);
5725 }
5726
5727 #if defined(CONFIG_USER_ONLY)
5728
5729 /* These should probably raise undefined insn exceptions.  */
5730 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
5731 {
5732     ARMCPU *cpu = arm_env_get_cpu(env);
5733
5734     cpu_abort(CPU(cpu), "v7m_msr %d\n", reg);
5735 }
5736
5737 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
5738 {
5739     ARMCPU *cpu = arm_env_get_cpu(env);
5740
5741     cpu_abort(CPU(cpu), "v7m_mrs %d\n", reg);
5742     return 0;
5743 }
5744
5745 void switch_mode(CPUARMState *env, int mode)
5746 {
5747     ARMCPU *cpu = arm_env_get_cpu(env);
5748
5749     if (mode != ARM_CPU_MODE_USR) {
5750         cpu_abort(CPU(cpu), "Tried to switch out of user mode\n");
5751     }
5752 }
5753
5754 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
5755                                  uint32_t cur_el, bool secure)
5756 {
5757     return 1;
5758 }
5759
5760 void aarch64_sync_64_to_32(CPUARMState *env)
5761 {
5762     g_assert_not_reached();
5763 }
5764
5765 #else
5766
5767 void switch_mode(CPUARMState *env, int mode)
5768 {
5769     int old_mode;
5770     int i;
5771
5772     old_mode = env->uncached_cpsr & CPSR_M;
5773     if (mode == old_mode)
5774         return;
5775
5776     if (old_mode == ARM_CPU_MODE_FIQ) {
5777         memcpy (env->fiq_regs, env->regs + 8, 5 * sizeof(uint32_t));
5778         memcpy (env->regs + 8, env->usr_regs, 5 * sizeof(uint32_t));
5779     } else if (mode == ARM_CPU_MODE_FIQ) {
5780         memcpy (env->usr_regs, env->regs + 8, 5 * sizeof(uint32_t));
5781         memcpy (env->regs + 8, env->fiq_regs, 5 * sizeof(uint32_t));
5782     }
5783
5784     i = bank_number(old_mode);
5785     env->banked_r13[i] = env->regs[13];
5786     env->banked_r14[i] = env->regs[14];
5787     env->banked_spsr[i] = env->spsr;
5788
5789     i = bank_number(mode);
5790     env->regs[13] = env->banked_r13[i];
5791     env->regs[14] = env->banked_r14[i];
5792     env->spsr = env->banked_spsr[i];
5793 }
5794
5795 /* Physical Interrupt Target EL Lookup Table
5796  *
5797  * [ From ARM ARM section G1.13.4 (Table G1-15) ]
5798  *
5799  * The below multi-dimensional table is used for looking up the target
5800  * exception level given numerous condition criteria.  Specifically, the
5801  * target EL is based on SCR and HCR routing controls as well as the
5802  * currently executing EL and secure state.
5803  *
5804  *    Dimensions:
5805  *    target_el_table[2][2][2][2][2][4]
5806  *                    |  |  |  |  |  +--- Current EL
5807  *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
5808  *                    |  |  |  +--------- HCR mask override
5809  *                    |  |  +------------ SCR exec state control
5810  *                    |  +--------------- SCR mask override
5811  *                    +------------------ 32-bit(0)/64-bit(1) EL3
5812  *
5813  *    The table values are as such:
5814  *    0-3 = EL0-EL3
5815  *     -1 = Cannot occur
5816  *
5817  * The ARM ARM target EL table includes entries indicating that an "exception
5818  * is not taken".  The two cases where this is applicable are:
5819  *    1) An exception is taken from EL3 but the SCR does not have the exception
5820  *    routed to EL3.
5821  *    2) An exception is taken from EL2 but the HCR does not have the exception
5822  *    routed to EL2.
5823  * In these two cases, the below table contain a target of EL1.  This value is
5824  * returned as it is expected that the consumer of the table data will check
5825  * for "target EL >= current EL" to ensure the exception is not taken.
5826  *
5827  *            SCR     HCR
5828  *         64  EA     AMO                 From
5829  *        BIT IRQ     IMO      Non-secure         Secure
5830  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
5831  */
5832 static const int8_t target_el_table[2][2][2][2][2][4] = {
5833     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
5834        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
5835       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
5836        {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
5837      {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
5838        {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
5839       {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
5840        {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
5841     {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
5842        {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},
5843       {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1, -1,  1 },},
5844        {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},},
5845      {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
5846        {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
5847       {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
5848        {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},},},
5849 };
5850
5851 /*
5852  * Determine the target EL for physical exceptions
5853  */
5854 uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
5855                                  uint32_t cur_el, bool secure)
5856 {
5857     CPUARMState *env = cs->env_ptr;
5858     int rw;
5859     int scr;
5860     int hcr;
5861     int target_el;
5862     /* Is the highest EL AArch64? */
5863     int is64 = arm_feature(env, ARM_FEATURE_AARCH64);
5864
5865     if (arm_feature(env, ARM_FEATURE_EL3)) {
5866         rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
5867     } else {
5868         /* Either EL2 is the highest EL (and so the EL2 register width
5869          * is given by is64); or there is no EL2 or EL3, in which case
5870          * the value of 'rw' does not affect the table lookup anyway.
5871          */
5872         rw = is64;
5873     }
5874
5875     switch (excp_idx) {
5876     case EXCP_IRQ:
5877         scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
5878         hcr = ((env->cp15.hcr_el2 & HCR_IMO) == HCR_IMO);
5879         break;
5880     case EXCP_FIQ:
5881         scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
5882         hcr = ((env->cp15.hcr_el2 & HCR_FMO) == HCR_FMO);
5883         break;
5884     default:
5885         scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
5886         hcr = ((env->cp15.hcr_el2 & HCR_AMO) == HCR_AMO);
5887         break;
5888     };
5889
5890     /* If HCR.TGE is set then HCR is treated as being 1 */
5891     hcr |= ((env->cp15.hcr_el2 & HCR_TGE) == HCR_TGE);
5892
5893     /* Perform a table-lookup for the target EL given the current state */
5894     target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
5895
5896     assert(target_el > 0);
5897
5898     return target_el;
5899 }
5900
5901 static void v7m_push(CPUARMState *env, uint32_t val)
5902 {
5903     CPUState *cs = CPU(arm_env_get_cpu(env));
5904
5905     env->regs[13] -= 4;
5906     stl_phys(cs->as, env->regs[13], val);
5907 }
5908
5909 static uint32_t v7m_pop(CPUARMState *env)
5910 {
5911     CPUState *cs = CPU(arm_env_get_cpu(env));
5912     uint32_t val;
5913
5914     val = ldl_phys(cs->as, env->regs[13]);
5915     env->regs[13] += 4;
5916     return val;
5917 }
5918
5919 /* Switch to V7M main or process stack pointer.  */
5920 static void switch_v7m_sp(CPUARMState *env, int process)
5921 {
5922     uint32_t tmp;
5923     if (env->v7m.current_sp != process) {
5924         tmp = env->v7m.other_sp;
5925         env->v7m.other_sp = env->regs[13];
5926         env->regs[13] = tmp;
5927         env->v7m.current_sp = process;
5928     }
5929 }
5930
5931 static void do_v7m_exception_exit(CPUARMState *env)
5932 {
5933     uint32_t type;
5934     uint32_t xpsr;
5935
5936     type = env->regs[15];
5937     if (env->v7m.exception != 0)
5938         armv7m_nvic_complete_irq(env->nvic, env->v7m.exception);
5939
5940     /* Switch to the target stack.  */
5941     switch_v7m_sp(env, (type & 4) != 0);
5942     /* Pop registers.  */
5943     env->regs[0] = v7m_pop(env);
5944     env->regs[1] = v7m_pop(env);
5945     env->regs[2] = v7m_pop(env);
5946     env->regs[3] = v7m_pop(env);
5947     env->regs[12] = v7m_pop(env);
5948     env->regs[14] = v7m_pop(env);
5949     env->regs[15] = v7m_pop(env);
5950     if (env->regs[15] & 1) {
5951         qemu_log_mask(LOG_GUEST_ERROR,
5952                       "M profile return from interrupt with misaligned "
5953                       "PC is UNPREDICTABLE\n");
5954         /* Actual hardware seems to ignore the lsbit, and there are several
5955          * RTOSes out there which incorrectly assume the r15 in the stack
5956          * frame should be a Thumb-style "lsbit indicates ARM/Thumb" value.
5957          */
5958         env->regs[15] &= ~1U;
5959     }
5960     xpsr = v7m_pop(env);
5961     xpsr_write(env, xpsr, 0xfffffdff);
5962     /* Undo stack alignment.  */
5963     if (xpsr & 0x200)
5964         env->regs[13] |= 4;
5965     /* ??? The exception return type specifies Thread/Handler mode.  However
5966        this is also implied by the xPSR value. Not sure what to do
5967        if there is a mismatch.  */
5968     /* ??? Likewise for mismatches between the CONTROL register and the stack
5969        pointer.  */
5970 }
5971
5972 static void arm_log_exception(int idx)
5973 {
5974     if (qemu_loglevel_mask(CPU_LOG_INT)) {
5975         const char *exc = NULL;
5976
5977         if (idx >= 0 && idx < ARRAY_SIZE(excnames)) {
5978             exc = excnames[idx];
5979         }
5980         if (!exc) {
5981             exc = "unknown";
5982         }
5983         qemu_log_mask(CPU_LOG_INT, "Taking exception %d [%s]\n", idx, exc);
5984     }
5985 }
5986
5987 void arm_v7m_cpu_do_interrupt(CPUState *cs)
5988 {
5989     ARMCPU *cpu = ARM_CPU(cs);
5990     CPUARMState *env = &cpu->env;
5991     uint32_t xpsr = xpsr_read(env);
5992     uint32_t lr;
5993     uint32_t addr;
5994
5995     arm_log_exception(cs->exception_index);
5996
5997     lr = 0xfffffff1;
5998     if (env->v7m.current_sp)
5999         lr |= 4;
6000     if (env->v7m.exception == 0)
6001         lr |= 8;
6002
6003     /* For exceptions we just mark as pending on the NVIC, and let that
6004        handle it.  */
6005     /* TODO: Need to escalate if the current priority is higher than the
6006        one we're raising.  */
6007     switch (cs->exception_index) {
6008     case EXCP_UDEF:
6009         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE);
6010         return;
6011     case EXCP_SWI:
6012         /* The PC already points to the next instruction.  */
6013         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SVC);
6014         return;
6015     case EXCP_PREFETCH_ABORT:
6016     case EXCP_DATA_ABORT:
6017         /* TODO: if we implemented the MPU registers, this is where we
6018          * should set the MMFAR, etc from exception.fsr and exception.vaddress.
6019          */
6020         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_MEM);
6021         return;
6022     case EXCP_BKPT:
6023         if (semihosting_enabled()) {
6024             int nr;
6025             nr = arm_lduw_code(env, env->regs[15], arm_sctlr_b(env)) & 0xff;
6026             if (nr == 0xab) {
6027                 env->regs[15] += 2;
6028                 qemu_log_mask(CPU_LOG_INT,
6029                               "...handling as semihosting call 0x%x\n",
6030                               env->regs[0]);
6031                 env->regs[0] = do_arm_semihosting(env);
6032                 return;
6033             }
6034         }
6035         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_DEBUG);
6036         return;
6037     case EXCP_IRQ:
6038         env->v7m.exception = armv7m_nvic_acknowledge_irq(env->nvic);
6039         break;
6040     case EXCP_EXCEPTION_EXIT:
6041         do_v7m_exception_exit(env);
6042         return;
6043     default:
6044         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
6045         return; /* Never happens.  Keep compiler happy.  */
6046     }
6047
6048     /* Align stack pointer.  */
6049     /* ??? Should only do this if Configuration Control Register
6050        STACKALIGN bit is set.  */
6051     if (env->regs[13] & 4) {
6052         env->regs[13] -= 4;
6053         xpsr |= 0x200;
6054     }
6055     /* Switch to the handler mode.  */
6056     v7m_push(env, xpsr);
6057     v7m_push(env, env->regs[15]);
6058     v7m_push(env, env->regs[14]);
6059     v7m_push(env, env->regs[12]);
6060     v7m_push(env, env->regs[3]);
6061     v7m_push(env, env->regs[2]);
6062     v7m_push(env, env->regs[1]);
6063     v7m_push(env, env->regs[0]);
6064     switch_v7m_sp(env, 0);
6065     /* Clear IT bits */
6066     env->condexec_bits = 0;
6067     env->regs[14] = lr;
6068     addr = ldl_phys(cs->as, env->v7m.vecbase + env->v7m.exception * 4);
6069     env->regs[15] = addr & 0xfffffffe;
6070     env->thumb = addr & 1;
6071 }
6072
6073 /* Function used to synchronize QEMU's AArch64 register set with AArch32
6074  * register set.  This is necessary when switching between AArch32 and AArch64
6075  * execution state.
6076  */
6077 void aarch64_sync_32_to_64(CPUARMState *env)
6078 {
6079     int i;
6080     uint32_t mode = env->uncached_cpsr & CPSR_M;
6081
6082     /* We can blanket copy R[0:7] to X[0:7] */
6083     for (i = 0; i < 8; i++) {
6084         env->xregs[i] = env->regs[i];
6085     }
6086
6087     /* Unless we are in FIQ mode, x8-x12 come from the user registers r8-r12.
6088      * Otherwise, they come from the banked user regs.
6089      */
6090     if (mode == ARM_CPU_MODE_FIQ) {
6091         for (i = 8; i < 13; i++) {
6092             env->xregs[i] = env->usr_regs[i - 8];
6093         }
6094     } else {
6095         for (i = 8; i < 13; i++) {
6096             env->xregs[i] = env->regs[i];
6097         }
6098     }
6099
6100     /* Registers x13-x23 are the various mode SP and FP registers. Registers
6101      * r13 and r14 are only copied if we are in that mode, otherwise we copy
6102      * from the mode banked register.
6103      */
6104     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
6105         env->xregs[13] = env->regs[13];
6106         env->xregs[14] = env->regs[14];
6107     } else {
6108         env->xregs[13] = env->banked_r13[bank_number(ARM_CPU_MODE_USR)];
6109         /* HYP is an exception in that it is copied from r14 */
6110         if (mode == ARM_CPU_MODE_HYP) {
6111             env->xregs[14] = env->regs[14];
6112         } else {
6113             env->xregs[14] = env->banked_r14[bank_number(ARM_CPU_MODE_USR)];
6114         }
6115     }
6116
6117     if (mode == ARM_CPU_MODE_HYP) {
6118         env->xregs[15] = env->regs[13];
6119     } else {
6120         env->xregs[15] = env->banked_r13[bank_number(ARM_CPU_MODE_HYP)];
6121     }
6122
6123     if (mode == ARM_CPU_MODE_IRQ) {
6124         env->xregs[16] = env->regs[14];
6125         env->xregs[17] = env->regs[13];
6126     } else {
6127         env->xregs[16] = env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)];
6128         env->xregs[17] = env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)];
6129     }
6130
6131     if (mode == ARM_CPU_MODE_SVC) {
6132         env->xregs[18] = env->regs[14];
6133         env->xregs[19] = env->regs[13];
6134     } else {
6135         env->xregs[18] = env->banked_r14[bank_number(ARM_CPU_MODE_SVC)];
6136         env->xregs[19] = env->banked_r13[bank_number(ARM_CPU_MODE_SVC)];
6137     }
6138
6139     if (mode == ARM_CPU_MODE_ABT) {
6140         env->xregs[20] = env->regs[14];
6141         env->xregs[21] = env->regs[13];
6142     } else {
6143         env->xregs[20] = env->banked_r14[bank_number(ARM_CPU_MODE_ABT)];
6144         env->xregs[21] = env->banked_r13[bank_number(ARM_CPU_MODE_ABT)];
6145     }
6146
6147     if (mode == ARM_CPU_MODE_UND) {
6148         env->xregs[22] = env->regs[14];
6149         env->xregs[23] = env->regs[13];
6150     } else {
6151         env->xregs[22] = env->banked_r14[bank_number(ARM_CPU_MODE_UND)];
6152         env->xregs[23] = env->banked_r13[bank_number(ARM_CPU_MODE_UND)];
6153     }
6154
6155     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
6156      * mode, then we can copy from r8-r14.  Otherwise, we copy from the
6157      * FIQ bank for r8-r14.
6158      */
6159     if (mode == ARM_CPU_MODE_FIQ) {
6160         for (i = 24; i < 31; i++) {
6161             env->xregs[i] = env->regs[i - 16];   /* X[24:30] <- R[8:14] */
6162         }
6163     } else {
6164         for (i = 24; i < 29; i++) {
6165             env->xregs[i] = env->fiq_regs[i - 24];
6166         }
6167         env->xregs[29] = env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)];
6168         env->xregs[30] = env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)];
6169     }
6170
6171     env->pc = env->regs[15];
6172 }
6173
6174 /* Function used to synchronize QEMU's AArch32 register set with AArch64
6175  * register set.  This is necessary when switching between AArch32 and AArch64
6176  * execution state.
6177  */
6178 void aarch64_sync_64_to_32(CPUARMState *env)
6179 {
6180     int i;
6181     uint32_t mode = env->uncached_cpsr & CPSR_M;
6182
6183     /* We can blanket copy X[0:7] to R[0:7] */
6184     for (i = 0; i < 8; i++) {
6185         env->regs[i] = env->xregs[i];
6186     }
6187
6188     /* Unless we are in FIQ mode, r8-r12 come from the user registers x8-x12.
6189      * Otherwise, we copy x8-x12 into the banked user regs.
6190      */
6191     if (mode == ARM_CPU_MODE_FIQ) {
6192         for (i = 8; i < 13; i++) {
6193             env->usr_regs[i - 8] = env->xregs[i];
6194         }
6195     } else {
6196         for (i = 8; i < 13; i++) {
6197             env->regs[i] = env->xregs[i];
6198         }
6199     }
6200
6201     /* Registers r13 & r14 depend on the current mode.
6202      * If we are in a given mode, we copy the corresponding x registers to r13
6203      * and r14.  Otherwise, we copy the x register to the banked r13 and r14
6204      * for the mode.
6205      */
6206     if (mode == ARM_CPU_MODE_USR || mode == ARM_CPU_MODE_SYS) {
6207         env->regs[13] = env->xregs[13];
6208         env->regs[14] = env->xregs[14];
6209     } else {
6210         env->banked_r13[bank_number(ARM_CPU_MODE_USR)] = env->xregs[13];
6211
6212         /* HYP is an exception in that it does not have its own banked r14 but
6213          * shares the USR r14
6214          */
6215         if (mode == ARM_CPU_MODE_HYP) {
6216             env->regs[14] = env->xregs[14];
6217         } else {
6218             env->banked_r14[bank_number(ARM_CPU_MODE_USR)] = env->xregs[14];
6219         }
6220     }
6221
6222     if (mode == ARM_CPU_MODE_HYP) {
6223         env->regs[13] = env->xregs[15];
6224     } else {
6225         env->banked_r13[bank_number(ARM_CPU_MODE_HYP)] = env->xregs[15];
6226     }
6227
6228     if (mode == ARM_CPU_MODE_IRQ) {
6229         env->regs[14] = env->xregs[16];
6230         env->regs[13] = env->xregs[17];
6231     } else {
6232         env->banked_r14[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[16];
6233         env->banked_r13[bank_number(ARM_CPU_MODE_IRQ)] = env->xregs[17];
6234     }
6235
6236     if (mode == ARM_CPU_MODE_SVC) {
6237         env->regs[14] = env->xregs[18];
6238         env->regs[13] = env->xregs[19];
6239     } else {
6240         env->banked_r14[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[18];
6241         env->banked_r13[bank_number(ARM_CPU_MODE_SVC)] = env->xregs[19];
6242     }
6243
6244     if (mode == ARM_CPU_MODE_ABT) {
6245         env->regs[14] = env->xregs[20];
6246         env->regs[13] = env->xregs[21];
6247     } else {
6248         env->banked_r14[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[20];
6249         env->banked_r13[bank_number(ARM_CPU_MODE_ABT)] = env->xregs[21];
6250     }
6251
6252     if (mode == ARM_CPU_MODE_UND) {
6253         env->regs[14] = env->xregs[22];
6254         env->regs[13] = env->xregs[23];
6255     } else {
6256         env->banked_r14[bank_number(ARM_CPU_MODE_UND)] = env->xregs[22];
6257         env->banked_r13[bank_number(ARM_CPU_MODE_UND)] = env->xregs[23];
6258     }
6259
6260     /* Registers x24-x30 are mapped to r8-r14 in FIQ mode.  If we are in FIQ
6261      * mode, then we can copy to r8-r14.  Otherwise, we copy to the
6262      * FIQ bank for r8-r14.
6263      */
6264     if (mode == ARM_CPU_MODE_FIQ) {
6265         for (i = 24; i < 31; i++) {
6266             env->regs[i - 16] = env->xregs[i];   /* X[24:30] -> R[8:14] */
6267         }
6268     } else {
6269         for (i = 24; i < 29; i++) {
6270             env->fiq_regs[i - 24] = env->xregs[i];
6271         }
6272         env->banked_r13[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[29];
6273         env->banked_r14[bank_number(ARM_CPU_MODE_FIQ)] = env->xregs[30];
6274     }
6275
6276     env->regs[15] = env->pc;
6277 }
6278
6279 static void arm_cpu_do_interrupt_aarch32(CPUState *cs)
6280 {
6281     ARMCPU *cpu = ARM_CPU(cs);
6282     CPUARMState *env = &cpu->env;
6283     uint32_t addr;
6284     uint32_t mask;
6285     int new_mode;
6286     uint32_t offset;
6287     uint32_t moe;
6288
6289     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
6290     switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
6291     case EC_BREAKPOINT:
6292     case EC_BREAKPOINT_SAME_EL:
6293         moe = 1;
6294         break;
6295     case EC_WATCHPOINT:
6296     case EC_WATCHPOINT_SAME_EL:
6297         moe = 10;
6298         break;
6299     case EC_AA32_BKPT:
6300         moe = 3;
6301         break;
6302     case EC_VECTORCATCH:
6303         moe = 5;
6304         break;
6305     default:
6306         moe = 0;
6307         break;
6308     }
6309
6310     if (moe) {
6311         env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
6312     }
6313
6314     /* TODO: Vectored interrupt controller.  */
6315     switch (cs->exception_index) {
6316     case EXCP_UDEF:
6317         new_mode = ARM_CPU_MODE_UND;
6318         addr = 0x04;
6319         mask = CPSR_I;
6320         if (env->thumb)
6321             offset = 2;
6322         else
6323             offset = 4;
6324         break;
6325     case EXCP_SWI:
6326         new_mode = ARM_CPU_MODE_SVC;
6327         addr = 0x08;
6328         mask = CPSR_I;
6329         /* The PC already points to the next instruction.  */
6330         offset = 0;
6331         break;
6332     case EXCP_BKPT:
6333         env->exception.fsr = 2;
6334         /* Fall through to prefetch abort.  */
6335     case EXCP_PREFETCH_ABORT:
6336         A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
6337         A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
6338         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
6339                       env->exception.fsr, (uint32_t)env->exception.vaddress);
6340         new_mode = ARM_CPU_MODE_ABT;
6341         addr = 0x0c;
6342         mask = CPSR_A | CPSR_I;
6343         offset = 4;
6344         break;
6345     case EXCP_DATA_ABORT:
6346         A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
6347         A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
6348         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
6349                       env->exception.fsr,
6350                       (uint32_t)env->exception.vaddress);
6351         new_mode = ARM_CPU_MODE_ABT;
6352         addr = 0x10;
6353         mask = CPSR_A | CPSR_I;
6354         offset = 8;
6355         break;
6356     case EXCP_IRQ:
6357         new_mode = ARM_CPU_MODE_IRQ;
6358         addr = 0x18;
6359         /* Disable IRQ and imprecise data aborts.  */
6360         mask = CPSR_A | CPSR_I;
6361         offset = 4;
6362         if (env->cp15.scr_el3 & SCR_IRQ) {
6363             /* IRQ routed to monitor mode */
6364             new_mode = ARM_CPU_MODE_MON;
6365             mask |= CPSR_F;
6366         }
6367         break;
6368     case EXCP_FIQ:
6369         new_mode = ARM_CPU_MODE_FIQ;
6370         addr = 0x1c;
6371         /* Disable FIQ, IRQ and imprecise data aborts.  */
6372         mask = CPSR_A | CPSR_I | CPSR_F;
6373         if (env->cp15.scr_el3 & SCR_FIQ) {
6374             /* FIQ routed to monitor mode */
6375             new_mode = ARM_CPU_MODE_MON;
6376         }
6377         offset = 4;
6378         break;
6379     case EXCP_SMC:
6380         new_mode = ARM_CPU_MODE_MON;
6381         addr = 0x08;
6382         mask = CPSR_A | CPSR_I | CPSR_F;
6383         offset = 0;
6384         break;
6385     default:
6386         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
6387         return; /* Never happens.  Keep compiler happy.  */
6388     }
6389
6390     if (new_mode == ARM_CPU_MODE_MON) {
6391         addr += env->cp15.mvbar;
6392     } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
6393         /* High vectors. When enabled, base address cannot be remapped. */
6394         addr += 0xffff0000;
6395     } else {
6396         /* ARM v7 architectures provide a vector base address register to remap
6397          * the interrupt vector table.
6398          * This register is only followed in non-monitor mode, and is banked.
6399          * Note: only bits 31:5 are valid.
6400          */
6401         addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
6402     }
6403
6404     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
6405         env->cp15.scr_el3 &= ~SCR_NS;
6406     }
6407
6408     switch_mode (env, new_mode);
6409     /* For exceptions taken to AArch32 we must clear the SS bit in both
6410      * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
6411      */
6412     env->uncached_cpsr &= ~PSTATE_SS;
6413     env->spsr = cpsr_read(env);
6414     /* Clear IT bits.  */
6415     env->condexec_bits = 0;
6416     /* Switch to the new mode, and to the correct instruction set.  */
6417     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
6418     /* Set new mode endianness */
6419     env->uncached_cpsr &= ~CPSR_E;
6420     if (env->cp15.sctlr_el[arm_current_el(env)] & SCTLR_EE) {
6421         env->uncached_cpsr |= ~CPSR_E;
6422     }
6423     env->daif |= mask;
6424     /* this is a lie, as the was no c1_sys on V4T/V5, but who cares
6425      * and we should just guard the thumb mode on V4 */
6426     if (arm_feature(env, ARM_FEATURE_V4T)) {
6427         env->thumb = (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
6428     }
6429     env->regs[14] = env->regs[15] + offset;
6430     env->regs[15] = addr;
6431 }
6432
6433 /* Handle exception entry to a target EL which is using AArch64 */
6434 static void arm_cpu_do_interrupt_aarch64(CPUState *cs)
6435 {
6436     ARMCPU *cpu = ARM_CPU(cs);
6437     CPUARMState *env = &cpu->env;
6438     unsigned int new_el = env->exception.target_el;
6439     target_ulong addr = env->cp15.vbar_el[new_el];
6440     unsigned int new_mode = aarch64_pstate_mode(new_el, true);
6441
6442     if (arm_current_el(env) < new_el) {
6443         /* Entry vector offset depends on whether the implemented EL
6444          * immediately lower than the target level is using AArch32 or AArch64
6445          */
6446         bool is_aa64;
6447
6448         switch (new_el) {
6449         case 3:
6450             is_aa64 = (env->cp15.scr_el3 & SCR_RW) != 0;
6451             break;
6452         case 2:
6453             is_aa64 = (env->cp15.hcr_el2 & HCR_RW) != 0;
6454             break;
6455         case 1:
6456             is_aa64 = is_a64(env);
6457             break;
6458         default:
6459             g_assert_not_reached();
6460         }
6461
6462         if (is_aa64) {
6463             addr += 0x400;
6464         } else {
6465             addr += 0x600;
6466         }
6467     } else if (pstate_read(env) & PSTATE_SP) {
6468         addr += 0x200;
6469     }
6470
6471     switch (cs->exception_index) {
6472     case EXCP_PREFETCH_ABORT:
6473     case EXCP_DATA_ABORT:
6474         env->cp15.far_el[new_el] = env->exception.vaddress;
6475         qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n",
6476                       env->cp15.far_el[new_el]);
6477         /* fall through */
6478     case EXCP_BKPT:
6479     case EXCP_UDEF:
6480     case EXCP_SWI:
6481     case EXCP_HVC:
6482     case EXCP_HYP_TRAP:
6483     case EXCP_SMC:
6484         env->cp15.esr_el[new_el] = env->exception.syndrome;
6485         break;
6486     case EXCP_IRQ:
6487     case EXCP_VIRQ:
6488         addr += 0x80;
6489         break;
6490     case EXCP_FIQ:
6491     case EXCP_VFIQ:
6492         addr += 0x100;
6493         break;
6494     case EXCP_SEMIHOST:
6495         qemu_log_mask(CPU_LOG_INT,
6496                       "...handling as semihosting call 0x%" PRIx64 "\n",
6497                       env->xregs[0]);
6498         env->xregs[0] = do_arm_semihosting(env);
6499         return;
6500     default:
6501         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
6502     }
6503
6504     if (is_a64(env)) {
6505         env->banked_spsr[aarch64_banked_spsr_index(new_el)] = pstate_read(env);
6506         aarch64_save_sp(env, arm_current_el(env));
6507         env->elr_el[new_el] = env->pc;
6508     } else {
6509         env->banked_spsr[aarch64_banked_spsr_index(new_el)] = cpsr_read(env);
6510         env->elr_el[new_el] = env->regs[15];
6511
6512         aarch64_sync_32_to_64(env);
6513
6514         env->condexec_bits = 0;
6515     }
6516     qemu_log_mask(CPU_LOG_INT, "...with ELR 0x%" PRIx64 "\n",
6517                   env->elr_el[new_el]);
6518
6519     pstate_write(env, PSTATE_DAIF | new_mode);
6520     env->aarch64 = 1;
6521     aarch64_restore_sp(env, new_el);
6522
6523     env->pc = addr;
6524
6525     qemu_log_mask(CPU_LOG_INT, "...to EL%d PC 0x%" PRIx64 " PSTATE 0x%x\n",
6526                   new_el, env->pc, pstate_read(env));
6527 }
6528
6529 static inline bool check_for_semihosting(CPUState *cs)
6530 {
6531     /* Check whether this exception is a semihosting call; if so
6532      * then handle it and return true; otherwise return false.
6533      */
6534     ARMCPU *cpu = ARM_CPU(cs);
6535     CPUARMState *env = &cpu->env;
6536
6537     if (is_a64(env)) {
6538         if (cs->exception_index == EXCP_SEMIHOST) {
6539             /* This is always the 64-bit semihosting exception.
6540              * The "is this usermode" and "is semihosting enabled"
6541              * checks have been done at translate time.
6542              */
6543             qemu_log_mask(CPU_LOG_INT,
6544                           "...handling as semihosting call 0x%" PRIx64 "\n",
6545                           env->xregs[0]);
6546             env->xregs[0] = do_arm_semihosting(env);
6547             return true;
6548         }
6549         return false;
6550     } else {
6551         uint32_t imm;
6552
6553         /* Only intercept calls from privileged modes, to provide some
6554          * semblance of security.
6555          */
6556         if (!semihosting_enabled() ||
6557             ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_USR)) {
6558             return false;
6559         }
6560
6561         switch (cs->exception_index) {
6562         case EXCP_SWI:
6563             /* Check for semihosting interrupt.  */
6564             if (env->thumb) {
6565                 imm = arm_lduw_code(env, env->regs[15] - 2, arm_sctlr_b(env))
6566                     & 0xff;
6567                 if (imm == 0xab) {
6568                     break;
6569                 }
6570             } else {
6571                 imm = arm_ldl_code(env, env->regs[15] - 4, arm_sctlr_b(env))
6572                     & 0xffffff;
6573                 if (imm == 0x123456) {
6574                     break;
6575                 }
6576             }
6577             return false;
6578         case EXCP_BKPT:
6579             /* See if this is a semihosting syscall.  */
6580             if (env->thumb) {
6581                 imm = arm_lduw_code(env, env->regs[15], arm_sctlr_b(env))
6582                     & 0xff;
6583                 if (imm == 0xab) {
6584                     env->regs[15] += 2;
6585                     break;
6586                 }
6587             }
6588             return false;
6589         default:
6590             return false;
6591         }
6592
6593         qemu_log_mask(CPU_LOG_INT,
6594                       "...handling as semihosting call 0x%x\n",
6595                       env->regs[0]);
6596         env->regs[0] = do_arm_semihosting(env);
6597         return true;
6598     }
6599 }
6600
6601 /* Handle a CPU exception for A and R profile CPUs.
6602  * Do any appropriate logging, handle PSCI calls, and then hand off
6603  * to the AArch64-entry or AArch32-entry function depending on the
6604  * target exception level's register width.
6605  */
6606 void arm_cpu_do_interrupt(CPUState *cs)
6607 {
6608     ARMCPU *cpu = ARM_CPU(cs);
6609     CPUARMState *env = &cpu->env;
6610     unsigned int new_el = env->exception.target_el;
6611
6612     assert(!IS_M(env));
6613
6614     arm_log_exception(cs->exception_index);
6615     qemu_log_mask(CPU_LOG_INT, "...from EL%d to EL%d\n", arm_current_el(env),
6616                   new_el);
6617     if (qemu_loglevel_mask(CPU_LOG_INT)
6618         && !excp_is_internal(cs->exception_index)) {
6619         qemu_log_mask(CPU_LOG_INT, "...with ESR %x/0x%" PRIx32 "\n",
6620                       env->exception.syndrome >> ARM_EL_EC_SHIFT,
6621                       env->exception.syndrome);
6622     }
6623
6624     if (arm_is_psci_call(cpu, cs->exception_index)) {
6625         arm_handle_psci_call(cpu);
6626         qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
6627         return;
6628     }
6629
6630     /* Semihosting semantics depend on the register width of the
6631      * code that caused the exception, not the target exception level,
6632      * so must be handled here.
6633      */
6634     if (check_for_semihosting(cs)) {
6635         return;
6636     }
6637
6638     assert(!excp_is_internal(cs->exception_index));
6639     if (arm_el_is_aa64(env, new_el)) {
6640         arm_cpu_do_interrupt_aarch64(cs);
6641     } else {
6642         arm_cpu_do_interrupt_aarch32(cs);
6643     }
6644
6645     arm_call_el_change_hook(cpu);
6646
6647     if (!kvm_enabled()) {
6648         cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
6649     }
6650 }
6651
6652 /* Return the exception level which controls this address translation regime */
6653 static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx)
6654 {
6655     switch (mmu_idx) {
6656     case ARMMMUIdx_S2NS:
6657     case ARMMMUIdx_S1E2:
6658         return 2;
6659     case ARMMMUIdx_S1E3:
6660         return 3;
6661     case ARMMMUIdx_S1SE0:
6662         return arm_el_is_aa64(env, 3) ? 1 : 3;
6663     case ARMMMUIdx_S1SE1:
6664     case ARMMMUIdx_S1NSE0:
6665     case ARMMMUIdx_S1NSE1:
6666         return 1;
6667     default:
6668         g_assert_not_reached();
6669     }
6670 }
6671
6672 /* Return true if this address translation regime is secure */
6673 static inline bool regime_is_secure(CPUARMState *env, ARMMMUIdx mmu_idx)
6674 {
6675     switch (mmu_idx) {
6676     case ARMMMUIdx_S12NSE0:
6677     case ARMMMUIdx_S12NSE1:
6678     case ARMMMUIdx_S1NSE0:
6679     case ARMMMUIdx_S1NSE1:
6680     case ARMMMUIdx_S1E2:
6681     case ARMMMUIdx_S2NS:
6682         return false;
6683     case ARMMMUIdx_S1E3:
6684     case ARMMMUIdx_S1SE0:
6685     case ARMMMUIdx_S1SE1:
6686         return true;
6687     default:
6688         g_assert_not_reached();
6689     }
6690 }
6691
6692 /* Return the SCTLR value which controls this address translation regime */
6693 static inline uint32_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx)
6694 {
6695     return env->cp15.sctlr_el[regime_el(env, mmu_idx)];
6696 }
6697
6698 /* Return true if the specified stage of address translation is disabled */
6699 static inline bool regime_translation_disabled(CPUARMState *env,
6700                                                ARMMMUIdx mmu_idx)
6701 {
6702     if (mmu_idx == ARMMMUIdx_S2NS) {
6703         return (env->cp15.hcr_el2 & HCR_VM) == 0;
6704     }
6705     return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0;
6706 }
6707
6708 static inline bool regime_translation_big_endian(CPUARMState *env,
6709                                                  ARMMMUIdx mmu_idx)
6710 {
6711     return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0;
6712 }
6713
6714 /* Return the TCR controlling this translation regime */
6715 static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx)
6716 {
6717     if (mmu_idx == ARMMMUIdx_S2NS) {
6718         return &env->cp15.vtcr_el2;
6719     }
6720     return &env->cp15.tcr_el[regime_el(env, mmu_idx)];
6721 }
6722
6723 /* Return the TTBR associated with this translation regime */
6724 static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx,
6725                                    int ttbrn)
6726 {
6727     if (mmu_idx == ARMMMUIdx_S2NS) {
6728         return env->cp15.vttbr_el2;
6729     }
6730     if (ttbrn == 0) {
6731         return env->cp15.ttbr0_el[regime_el(env, mmu_idx)];
6732     } else {
6733         return env->cp15.ttbr1_el[regime_el(env, mmu_idx)];
6734     }
6735 }
6736
6737 /* Return true if the translation regime is using LPAE format page tables */
6738 static inline bool regime_using_lpae_format(CPUARMState *env,
6739                                             ARMMMUIdx mmu_idx)
6740 {
6741     int el = regime_el(env, mmu_idx);
6742     if (el == 2 || arm_el_is_aa64(env, el)) {
6743         return true;
6744     }
6745     if (arm_feature(env, ARM_FEATURE_LPAE)
6746         && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) {
6747         return true;
6748     }
6749     return false;
6750 }
6751
6752 /* Returns true if the stage 1 translation regime is using LPAE format page
6753  * tables. Used when raising alignment exceptions, whose FSR changes depending
6754  * on whether the long or short descriptor format is in use. */
6755 bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx)
6756 {
6757     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
6758         mmu_idx += ARMMMUIdx_S1NSE0;
6759     }
6760
6761     return regime_using_lpae_format(env, mmu_idx);
6762 }
6763
6764 static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx)
6765 {
6766     switch (mmu_idx) {
6767     case ARMMMUIdx_S1SE0:
6768     case ARMMMUIdx_S1NSE0:
6769         return true;
6770     default:
6771         return false;
6772     case ARMMMUIdx_S12NSE0:
6773     case ARMMMUIdx_S12NSE1:
6774         g_assert_not_reached();
6775     }
6776 }
6777
6778 /* Translate section/page access permissions to page
6779  * R/W protection flags
6780  *
6781  * @env:         CPUARMState
6782  * @mmu_idx:     MMU index indicating required translation regime
6783  * @ap:          The 3-bit access permissions (AP[2:0])
6784  * @domain_prot: The 2-bit domain access permissions
6785  */
6786 static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx,
6787                                 int ap, int domain_prot)
6788 {
6789     bool is_user = regime_is_user(env, mmu_idx);
6790
6791     if (domain_prot == 3) {
6792         return PAGE_READ | PAGE_WRITE;
6793     }
6794
6795     switch (ap) {
6796     case 0:
6797         if (arm_feature(env, ARM_FEATURE_V7)) {
6798             return 0;
6799         }
6800         switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) {
6801         case SCTLR_S:
6802             return is_user ? 0 : PAGE_READ;
6803         case SCTLR_R:
6804             return PAGE_READ;
6805         default:
6806             return 0;
6807         }
6808     case 1:
6809         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
6810     case 2:
6811         if (is_user) {
6812             return PAGE_READ;
6813         } else {
6814             return PAGE_READ | PAGE_WRITE;
6815         }
6816     case 3:
6817         return PAGE_READ | PAGE_WRITE;
6818     case 4: /* Reserved.  */
6819         return 0;
6820     case 5:
6821         return is_user ? 0 : PAGE_READ;
6822     case 6:
6823         return PAGE_READ;
6824     case 7:
6825         if (!arm_feature(env, ARM_FEATURE_V6K)) {
6826             return 0;
6827         }
6828         return PAGE_READ;
6829     default:
6830         g_assert_not_reached();
6831     }
6832 }
6833
6834 /* Translate section/page access permissions to page
6835  * R/W protection flags.
6836  *
6837  * @ap:      The 2-bit simple AP (AP[2:1])
6838  * @is_user: TRUE if accessing from PL0
6839  */
6840 static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user)
6841 {
6842     switch (ap) {
6843     case 0:
6844         return is_user ? 0 : PAGE_READ | PAGE_WRITE;
6845     case 1:
6846         return PAGE_READ | PAGE_WRITE;
6847     case 2:
6848         return is_user ? 0 : PAGE_READ;
6849     case 3:
6850         return PAGE_READ;
6851     default:
6852         g_assert_not_reached();
6853     }
6854 }
6855
6856 static inline int
6857 simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap)
6858 {
6859     return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx));
6860 }
6861
6862 /* Translate S2 section/page access permissions to protection flags
6863  *
6864  * @env:     CPUARMState
6865  * @s2ap:    The 2-bit stage2 access permissions (S2AP)
6866  * @xn:      XN (execute-never) bit
6867  */
6868 static int get_S2prot(CPUARMState *env, int s2ap, int xn)
6869 {
6870     int prot = 0;
6871
6872     if (s2ap & 1) {
6873         prot |= PAGE_READ;
6874     }
6875     if (s2ap & 2) {
6876         prot |= PAGE_WRITE;
6877     }
6878     if (!xn) {
6879         if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) {
6880             prot |= PAGE_EXEC;
6881         }
6882     }
6883     return prot;
6884 }
6885
6886 /* Translate section/page access permissions to protection flags
6887  *
6888  * @env:     CPUARMState
6889  * @mmu_idx: MMU index indicating required translation regime
6890  * @is_aa64: TRUE if AArch64
6891  * @ap:      The 2-bit simple AP (AP[2:1])
6892  * @ns:      NS (non-secure) bit
6893  * @xn:      XN (execute-never) bit
6894  * @pxn:     PXN (privileged execute-never) bit
6895  */
6896 static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64,
6897                       int ap, int ns, int xn, int pxn)
6898 {
6899     bool is_user = regime_is_user(env, mmu_idx);
6900     int prot_rw, user_rw;
6901     bool have_wxn;
6902     int wxn = 0;
6903
6904     assert(mmu_idx != ARMMMUIdx_S2NS);
6905
6906     user_rw = simple_ap_to_rw_prot_is_user(ap, true);
6907     if (is_user) {
6908         prot_rw = user_rw;
6909     } else {
6910         prot_rw = simple_ap_to_rw_prot_is_user(ap, false);
6911     }
6912
6913     if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) {
6914         return prot_rw;
6915     }
6916
6917     /* TODO have_wxn should be replaced with
6918      *   ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2)
6919      * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE
6920      * compatible processors have EL2, which is required for [U]WXN.
6921      */
6922     have_wxn = arm_feature(env, ARM_FEATURE_LPAE);
6923
6924     if (have_wxn) {
6925         wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN;
6926     }
6927
6928     if (is_aa64) {
6929         switch (regime_el(env, mmu_idx)) {
6930         case 1:
6931             if (!is_user) {
6932                 xn = pxn || (user_rw & PAGE_WRITE);
6933             }
6934             break;
6935         case 2:
6936         case 3:
6937             break;
6938         }
6939     } else if (arm_feature(env, ARM_FEATURE_V7)) {
6940         switch (regime_el(env, mmu_idx)) {
6941         case 1:
6942         case 3:
6943             if (is_user) {
6944                 xn = xn || !(user_rw & PAGE_READ);
6945             } else {
6946                 int uwxn = 0;
6947                 if (have_wxn) {
6948                     uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN;
6949                 }
6950                 xn = xn || !(prot_rw & PAGE_READ) || pxn ||
6951                      (uwxn && (user_rw & PAGE_WRITE));
6952             }
6953             break;
6954         case 2:
6955             break;
6956         }
6957     } else {
6958         xn = wxn = 0;
6959     }
6960
6961     if (xn || (wxn && (prot_rw & PAGE_WRITE))) {
6962         return prot_rw;
6963     }
6964     return prot_rw | PAGE_EXEC;
6965 }
6966
6967 static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx,
6968                                      uint32_t *table, uint32_t address)
6969 {
6970     /* Note that we can only get here for an AArch32 PL0/PL1 lookup */
6971     TCR *tcr = regime_tcr(env, mmu_idx);
6972
6973     if (address & tcr->mask) {
6974         if (tcr->raw_tcr & TTBCR_PD1) {
6975             /* Translation table walk disabled for TTBR1 */
6976             return false;
6977         }
6978         *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000;
6979     } else {
6980         if (tcr->raw_tcr & TTBCR_PD0) {
6981             /* Translation table walk disabled for TTBR0 */
6982             return false;
6983         }
6984         *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask;
6985     }
6986     *table |= (address >> 18) & 0x3ffc;
6987     return true;
6988 }
6989
6990 /* Translate a S1 pagetable walk through S2 if needed.  */
6991 static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx,
6992                                hwaddr addr, MemTxAttrs txattrs,
6993                                uint32_t *fsr,
6994                                ARMMMUFaultInfo *fi)
6995 {
6996     if ((mmu_idx == ARMMMUIdx_S1NSE0 || mmu_idx == ARMMMUIdx_S1NSE1) &&
6997         !regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
6998         target_ulong s2size;
6999         hwaddr s2pa;
7000         int s2prot;
7001         int ret;
7002
7003         ret = get_phys_addr_lpae(env, addr, 0, ARMMMUIdx_S2NS, &s2pa,
7004                                  &txattrs, &s2prot, &s2size, fsr, fi);
7005         if (ret) {
7006             fi->s2addr = addr;
7007             fi->stage2 = true;
7008             fi->s1ptw = true;
7009             return ~0;
7010         }
7011         addr = s2pa;
7012     }
7013     return addr;
7014 }
7015
7016 /* All loads done in the course of a page table walk go through here.
7017  * TODO: rather than ignoring errors from physical memory reads (which
7018  * are external aborts in ARM terminology) we should propagate this
7019  * error out so that we can turn it into a Data Abort if this walk
7020  * was being done for a CPU load/store or an address translation instruction
7021  * (but not if it was for a debug access).
7022  */
7023 static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure,
7024                             ARMMMUIdx mmu_idx, uint32_t *fsr,
7025                             ARMMMUFaultInfo *fi)
7026 {
7027     ARMCPU *cpu = ARM_CPU(cs);
7028     CPUARMState *env = &cpu->env;
7029     MemTxAttrs attrs = {};
7030     AddressSpace *as;
7031
7032     attrs.secure = is_secure;
7033     as = arm_addressspace(cs, attrs);
7034     addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
7035     if (fi->s1ptw) {
7036         return 0;
7037     }
7038     if (regime_translation_big_endian(env, mmu_idx)) {
7039         return address_space_ldl_be(as, addr, attrs, NULL);
7040     } else {
7041         return address_space_ldl_le(as, addr, attrs, NULL);
7042     }
7043 }
7044
7045 static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure,
7046                             ARMMMUIdx mmu_idx, uint32_t *fsr,
7047                             ARMMMUFaultInfo *fi)
7048 {
7049     ARMCPU *cpu = ARM_CPU(cs);
7050     CPUARMState *env = &cpu->env;
7051     MemTxAttrs attrs = {};
7052     AddressSpace *as;
7053
7054     attrs.secure = is_secure;
7055     as = arm_addressspace(cs, attrs);
7056     addr = S1_ptw_translate(env, mmu_idx, addr, attrs, fsr, fi);
7057     if (fi->s1ptw) {
7058         return 0;
7059     }
7060     if (regime_translation_big_endian(env, mmu_idx)) {
7061         return address_space_ldq_be(as, addr, attrs, NULL);
7062     } else {
7063         return address_space_ldq_le(as, addr, attrs, NULL);
7064     }
7065 }
7066
7067 static bool get_phys_addr_v5(CPUARMState *env, uint32_t address,
7068                              int access_type, ARMMMUIdx mmu_idx,
7069                              hwaddr *phys_ptr, int *prot,
7070                              target_ulong *page_size, uint32_t *fsr,
7071                              ARMMMUFaultInfo *fi)
7072 {
7073     CPUState *cs = CPU(arm_env_get_cpu(env));
7074     int code;
7075     uint32_t table;
7076     uint32_t desc;
7077     int type;
7078     int ap;
7079     int domain = 0;
7080     int domain_prot;
7081     hwaddr phys_addr;
7082     uint32_t dacr;
7083
7084     /* Pagetable walk.  */
7085     /* Lookup l1 descriptor.  */
7086     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
7087         /* Section translation fault if page walk is disabled by PD0 or PD1 */
7088         code = 5;
7089         goto do_fault;
7090     }
7091     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
7092                        mmu_idx, fsr, fi);
7093     type = (desc & 3);
7094     domain = (desc >> 5) & 0x0f;
7095     if (regime_el(env, mmu_idx) == 1) {
7096         dacr = env->cp15.dacr_ns;
7097     } else {
7098         dacr = env->cp15.dacr_s;
7099     }
7100     domain_prot = (dacr >> (domain * 2)) & 3;
7101     if (type == 0) {
7102         /* Section translation fault.  */
7103         code = 5;
7104         goto do_fault;
7105     }
7106     if (domain_prot == 0 || domain_prot == 2) {
7107         if (type == 2)
7108             code = 9; /* Section domain fault.  */
7109         else
7110             code = 11; /* Page domain fault.  */
7111         goto do_fault;
7112     }
7113     if (type == 2) {
7114         /* 1Mb section.  */
7115         phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
7116         ap = (desc >> 10) & 3;
7117         code = 13;
7118         *page_size = 1024 * 1024;
7119     } else {
7120         /* Lookup l2 entry.  */
7121         if (type == 1) {
7122             /* Coarse pagetable.  */
7123             table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
7124         } else {
7125             /* Fine pagetable.  */
7126             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
7127         }
7128         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
7129                            mmu_idx, fsr, fi);
7130         switch (desc & 3) {
7131         case 0: /* Page translation fault.  */
7132             code = 7;
7133             goto do_fault;
7134         case 1: /* 64k page.  */
7135             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
7136             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
7137             *page_size = 0x10000;
7138             break;
7139         case 2: /* 4k page.  */
7140             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
7141             ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
7142             *page_size = 0x1000;
7143             break;
7144         case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */
7145             if (type == 1) {
7146                 /* ARMv6/XScale extended small page format */
7147                 if (arm_feature(env, ARM_FEATURE_XSCALE)
7148                     || arm_feature(env, ARM_FEATURE_V6)) {
7149                     phys_addr = (desc & 0xfffff000) | (address & 0xfff);
7150                     *page_size = 0x1000;
7151                 } else {
7152                     /* UNPREDICTABLE in ARMv5; we choose to take a
7153                      * page translation fault.
7154                      */
7155                     code = 7;
7156                     goto do_fault;
7157                 }
7158             } else {
7159                 phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
7160                 *page_size = 0x400;
7161             }
7162             ap = (desc >> 4) & 3;
7163             break;
7164         default:
7165             /* Never happens, but compiler isn't smart enough to tell.  */
7166             abort();
7167         }
7168         code = 15;
7169     }
7170     *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
7171     *prot |= *prot ? PAGE_EXEC : 0;
7172     if (!(*prot & (1 << access_type))) {
7173         /* Access permission fault.  */
7174         goto do_fault;
7175     }
7176     *phys_ptr = phys_addr;
7177     return false;
7178 do_fault:
7179     *fsr = code | (domain << 4);
7180     return true;
7181 }
7182
7183 static bool get_phys_addr_v6(CPUARMState *env, uint32_t address,
7184                              int access_type, ARMMMUIdx mmu_idx,
7185                              hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
7186                              target_ulong *page_size, uint32_t *fsr,
7187                              ARMMMUFaultInfo *fi)
7188 {
7189     CPUState *cs = CPU(arm_env_get_cpu(env));
7190     int code;
7191     uint32_t table;
7192     uint32_t desc;
7193     uint32_t xn;
7194     uint32_t pxn = 0;
7195     int type;
7196     int ap;
7197     int domain = 0;
7198     int domain_prot;
7199     hwaddr phys_addr;
7200     uint32_t dacr;
7201     bool ns;
7202
7203     /* Pagetable walk.  */
7204     /* Lookup l1 descriptor.  */
7205     if (!get_level1_table_address(env, mmu_idx, &table, address)) {
7206         /* Section translation fault if page walk is disabled by PD0 or PD1 */
7207         code = 5;
7208         goto do_fault;
7209     }
7210     desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
7211                        mmu_idx, fsr, fi);
7212     type = (desc & 3);
7213     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
7214         /* Section translation fault, or attempt to use the encoding
7215          * which is Reserved on implementations without PXN.
7216          */
7217         code = 5;
7218         goto do_fault;
7219     }
7220     if ((type == 1) || !(desc & (1 << 18))) {
7221         /* Page or Section.  */
7222         domain = (desc >> 5) & 0x0f;
7223     }
7224     if (regime_el(env, mmu_idx) == 1) {
7225         dacr = env->cp15.dacr_ns;
7226     } else {
7227         dacr = env->cp15.dacr_s;
7228     }
7229     domain_prot = (dacr >> (domain * 2)) & 3;
7230     if (domain_prot == 0 || domain_prot == 2) {
7231         if (type != 1) {
7232             code = 9; /* Section domain fault.  */
7233         } else {
7234             code = 11; /* Page domain fault.  */
7235         }
7236         goto do_fault;
7237     }
7238     if (type != 1) {
7239         if (desc & (1 << 18)) {
7240             /* Supersection.  */
7241             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
7242             phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32;
7243             phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36;
7244             *page_size = 0x1000000;
7245         } else {
7246             /* Section.  */
7247             phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
7248             *page_size = 0x100000;
7249         }
7250         ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
7251         xn = desc & (1 << 4);
7252         pxn = desc & 1;
7253         code = 13;
7254         ns = extract32(desc, 19, 1);
7255     } else {
7256         if (arm_feature(env, ARM_FEATURE_PXN)) {
7257             pxn = (desc >> 2) & 1;
7258         }
7259         ns = extract32(desc, 3, 1);
7260         /* Lookup l2 entry.  */
7261         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
7262         desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx),
7263                            mmu_idx, fsr, fi);
7264         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
7265         switch (desc & 3) {
7266         case 0: /* Page translation fault.  */
7267             code = 7;
7268             goto do_fault;
7269         case 1: /* 64k page.  */
7270             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
7271             xn = desc & (1 << 15);
7272             *page_size = 0x10000;
7273             break;
7274         case 2: case 3: /* 4k page.  */
7275             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
7276             xn = desc & 1;
7277             *page_size = 0x1000;
7278             break;
7279         default:
7280             /* Never happens, but compiler isn't smart enough to tell.  */
7281             abort();
7282         }
7283         code = 15;
7284     }
7285     if (domain_prot == 3) {
7286         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
7287     } else {
7288         if (pxn && !regime_is_user(env, mmu_idx)) {
7289             xn = 1;
7290         }
7291         if (xn && access_type == 2)
7292             goto do_fault;
7293
7294         if (arm_feature(env, ARM_FEATURE_V6K) &&
7295                 (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) {
7296             /* The simplified model uses AP[0] as an access control bit.  */
7297             if ((ap & 1) == 0) {
7298                 /* Access flag fault.  */
7299                 code = (code == 15) ? 6 : 3;
7300                 goto do_fault;
7301             }
7302             *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1);
7303         } else {
7304             *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot);
7305         }
7306         if (*prot && !xn) {
7307             *prot |= PAGE_EXEC;
7308         }
7309         if (!(*prot & (1 << access_type))) {
7310             /* Access permission fault.  */
7311             goto do_fault;
7312         }
7313     }
7314     if (ns) {
7315         /* The NS bit will (as required by the architecture) have no effect if
7316          * the CPU doesn't support TZ or this is a non-secure translation
7317          * regime, because the attribute will already be non-secure.
7318          */
7319         attrs->secure = false;
7320     }
7321     *phys_ptr = phys_addr;
7322     return false;
7323 do_fault:
7324     *fsr = code | (domain << 4);
7325     return true;
7326 }
7327
7328 /* Fault type for long-descriptor MMU fault reporting; this corresponds
7329  * to bits [5..2] in the STATUS field in long-format DFSR/IFSR.
7330  */
7331 typedef enum {
7332     translation_fault = 1,
7333     access_fault = 2,
7334     permission_fault = 3,
7335 } MMUFaultType;
7336
7337 /*
7338  * check_s2_mmu_setup
7339  * @cpu:        ARMCPU
7340  * @is_aa64:    True if the translation regime is in AArch64 state
7341  * @startlevel: Suggested starting level
7342  * @inputsize:  Bitsize of IPAs
7343  * @stride:     Page-table stride (See the ARM ARM)
7344  *
7345  * Returns true if the suggested S2 translation parameters are OK and
7346  * false otherwise.
7347  */
7348 static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level,
7349                                int inputsize, int stride)
7350 {
7351     const int grainsize = stride + 3;
7352     int startsizecheck;
7353
7354     /* Negative levels are never allowed.  */
7355     if (level < 0) {
7356         return false;
7357     }
7358
7359     startsizecheck = inputsize - ((3 - level) * stride + grainsize);
7360     if (startsizecheck < 1 || startsizecheck > stride + 4) {
7361         return false;
7362     }
7363
7364     if (is_aa64) {
7365         CPUARMState *env = &cpu->env;
7366         unsigned int pamax = arm_pamax(cpu);
7367
7368         switch (stride) {
7369         case 13: /* 64KB Pages.  */
7370             if (level == 0 || (level == 1 && pamax <= 42)) {
7371                 return false;
7372             }
7373             break;
7374         case 11: /* 16KB Pages.  */
7375             if (level == 0 || (level == 1 && pamax <= 40)) {
7376                 return false;
7377             }
7378             break;
7379         case 9: /* 4KB Pages.  */
7380             if (level == 0 && pamax <= 42) {
7381                 return false;
7382             }
7383             break;
7384         default:
7385             g_assert_not_reached();
7386         }
7387
7388         /* Inputsize checks.  */
7389         if (inputsize > pamax &&
7390             (arm_el_is_aa64(env, 1) || inputsize > 40)) {
7391             /* This is CONSTRAINED UNPREDICTABLE and we choose to fault.  */
7392             return false;
7393         }
7394     } else {
7395         /* AArch32 only supports 4KB pages. Assert on that.  */
7396         assert(stride == 9);
7397
7398         if (level == 0) {
7399             return false;
7400         }
7401     }
7402     return true;
7403 }
7404
7405 static bool get_phys_addr_lpae(CPUARMState *env, target_ulong address,
7406                                int access_type, ARMMMUIdx mmu_idx,
7407                                hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot,
7408                                target_ulong *page_size_ptr, uint32_t *fsr,
7409                                ARMMMUFaultInfo *fi)
7410 {
7411     ARMCPU *cpu = arm_env_get_cpu(env);
7412     CPUState *cs = CPU(cpu);
7413     /* Read an LPAE long-descriptor translation table. */
7414     MMUFaultType fault_type = translation_fault;
7415     uint32_t level;
7416     uint32_t epd = 0;
7417     int32_t t0sz, t1sz;
7418     uint32_t tg;
7419     uint64_t ttbr;
7420     int ttbr_select;
7421     hwaddr descaddr, indexmask, indexmask_grainsize;
7422     uint32_t tableattrs;
7423     target_ulong page_size;
7424     uint32_t attrs;
7425     int32_t stride = 9;
7426     int32_t addrsize;
7427     int inputsize;
7428     int32_t tbi = 0;
7429     TCR *tcr = regime_tcr(env, mmu_idx);
7430     int ap, ns, xn, pxn;
7431     uint32_t el = regime_el(env, mmu_idx);
7432     bool ttbr1_valid = true;
7433     uint64_t descaddrmask;
7434     bool aarch64 = arm_el_is_aa64(env, el);
7435
7436     /* TODO:
7437      * This code does not handle the different format TCR for VTCR_EL2.
7438      * This code also does not support shareability levels.
7439      * Attribute and permission bit handling should also be checked when adding
7440      * support for those page table walks.
7441      */
7442     if (aarch64) {
7443         level = 0;
7444         addrsize = 64;
7445         if (el > 1) {
7446             if (mmu_idx != ARMMMUIdx_S2NS) {
7447                 tbi = extract64(tcr->raw_tcr, 20, 1);
7448             }
7449         } else {
7450             if (extract64(address, 55, 1)) {
7451                 tbi = extract64(tcr->raw_tcr, 38, 1);
7452             } else {
7453                 tbi = extract64(tcr->raw_tcr, 37, 1);
7454             }
7455         }
7456         tbi *= 8;
7457
7458         /* If we are in 64-bit EL2 or EL3 then there is no TTBR1, so mark it
7459          * invalid.
7460          */
7461         if (el > 1) {
7462             ttbr1_valid = false;
7463         }
7464     } else {
7465         level = 1;
7466         addrsize = 32;
7467         /* There is no TTBR1 for EL2 */
7468         if (el == 2) {
7469             ttbr1_valid = false;
7470         }
7471     }
7472
7473     /* Determine whether this address is in the region controlled by
7474      * TTBR0 or TTBR1 (or if it is in neither region and should fault).
7475      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
7476      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
7477      */
7478     if (aarch64) {
7479         /* AArch64 translation.  */
7480         t0sz = extract32(tcr->raw_tcr, 0, 6);
7481         t0sz = MIN(t0sz, 39);
7482         t0sz = MAX(t0sz, 16);
7483     } else if (mmu_idx != ARMMMUIdx_S2NS) {
7484         /* AArch32 stage 1 translation.  */
7485         t0sz = extract32(tcr->raw_tcr, 0, 3);
7486     } else {
7487         /* AArch32 stage 2 translation.  */
7488         bool sext = extract32(tcr->raw_tcr, 4, 1);
7489         bool sign = extract32(tcr->raw_tcr, 3, 1);
7490         /* Address size is 40-bit for a stage 2 translation,
7491          * and t0sz can be negative (from -8 to 7),
7492          * so we need to adjust it to use the TTBR selecting logic below.
7493          */
7494         addrsize = 40;
7495         t0sz = sextract32(tcr->raw_tcr, 0, 4) + 8;
7496
7497         /* If the sign-extend bit is not the same as t0sz[3], the result
7498          * is unpredictable. Flag this as a guest error.  */
7499         if (sign != sext) {
7500             qemu_log_mask(LOG_GUEST_ERROR,
7501                           "AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n");
7502         }
7503     }
7504     t1sz = extract32(tcr->raw_tcr, 16, 6);
7505     if (aarch64) {
7506         t1sz = MIN(t1sz, 39);
7507         t1sz = MAX(t1sz, 16);
7508     }
7509     if (t0sz && !extract64(address, addrsize - t0sz, t0sz - tbi)) {
7510         /* there is a ttbr0 region and we are in it (high bits all zero) */
7511         ttbr_select = 0;
7512     } else if (ttbr1_valid && t1sz &&
7513                !extract64(~address, addrsize - t1sz, t1sz - tbi)) {
7514         /* there is a ttbr1 region and we are in it (high bits all one) */
7515         ttbr_select = 1;
7516     } else if (!t0sz) {
7517         /* ttbr0 region is "everything not in the ttbr1 region" */
7518         ttbr_select = 0;
7519     } else if (!t1sz && ttbr1_valid) {
7520         /* ttbr1 region is "everything not in the ttbr0 region" */
7521         ttbr_select = 1;
7522     } else {
7523         /* in the gap between the two regions, this is a Translation fault */
7524         fault_type = translation_fault;
7525         goto do_fault;
7526     }
7527
7528     /* Note that QEMU ignores shareability and cacheability attributes,
7529      * so we don't need to do anything with the SH, ORGN, IRGN fields
7530      * in the TTBCR.  Similarly, TTBCR:A1 selects whether we get the
7531      * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
7532      * implement any ASID-like capability so we can ignore it (instead
7533      * we will always flush the TLB any time the ASID is changed).
7534      */
7535     if (ttbr_select == 0) {
7536         ttbr = regime_ttbr(env, mmu_idx, 0);
7537         if (el < 2) {
7538             epd = extract32(tcr->raw_tcr, 7, 1);
7539         }
7540         inputsize = addrsize - t0sz;
7541
7542         tg = extract32(tcr->raw_tcr, 14, 2);
7543         if (tg == 1) { /* 64KB pages */
7544             stride = 13;
7545         }
7546         if (tg == 2) { /* 16KB pages */
7547             stride = 11;
7548         }
7549     } else {
7550         /* We should only be here if TTBR1 is valid */
7551         assert(ttbr1_valid);
7552
7553         ttbr = regime_ttbr(env, mmu_idx, 1);
7554         epd = extract32(tcr->raw_tcr, 23, 1);
7555         inputsize = addrsize - t1sz;
7556
7557         tg = extract32(tcr->raw_tcr, 30, 2);
7558         if (tg == 3)  { /* 64KB pages */
7559             stride = 13;
7560         }
7561         if (tg == 1) { /* 16KB pages */
7562             stride = 11;
7563         }
7564     }
7565
7566     /* Here we should have set up all the parameters for the translation:
7567      * inputsize, ttbr, epd, stride, tbi
7568      */
7569
7570     if (epd) {
7571         /* Translation table walk disabled => Translation fault on TLB miss
7572          * Note: This is always 0 on 64-bit EL2 and EL3.
7573          */
7574         goto do_fault;
7575     }
7576
7577     if (mmu_idx != ARMMMUIdx_S2NS) {
7578         /* The starting level depends on the virtual address size (which can
7579          * be up to 48 bits) and the translation granule size. It indicates
7580          * the number of strides (stride bits at a time) needed to
7581          * consume the bits of the input address. In the pseudocode this is:
7582          *  level = 4 - RoundUp((inputsize - grainsize) / stride)
7583          * where their 'inputsize' is our 'inputsize', 'grainsize' is
7584          * our 'stride + 3' and 'stride' is our 'stride'.
7585          * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
7586          * = 4 - (inputsize - stride - 3 + stride - 1) / stride
7587          * = 4 - (inputsize - 4) / stride;
7588          */
7589         level = 4 - (inputsize - 4) / stride;
7590     } else {
7591         /* For stage 2 translations the starting level is specified by the
7592          * VTCR_EL2.SL0 field (whose interpretation depends on the page size)
7593          */
7594         uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2);
7595         uint32_t startlevel;
7596         bool ok;
7597
7598         if (!aarch64 || stride == 9) {
7599             /* AArch32 or 4KB pages */
7600             startlevel = 2 - sl0;
7601         } else {
7602             /* 16KB or 64KB pages */
7603             startlevel = 3 - sl0;
7604         }
7605
7606         /* Check that the starting level is valid. */
7607         ok = check_s2_mmu_setup(cpu, aarch64, startlevel,
7608                                 inputsize, stride);
7609         if (!ok) {
7610             fault_type = translation_fault;
7611             goto do_fault;
7612         }
7613         level = startlevel;
7614     }
7615
7616     indexmask_grainsize = (1ULL << (stride + 3)) - 1;
7617     indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1;
7618
7619     /* Now we can extract the actual base address from the TTBR */
7620     descaddr = extract64(ttbr, 0, 48);
7621     descaddr &= ~indexmask;
7622
7623     /* The address field in the descriptor goes up to bit 39 for ARMv7
7624      * but up to bit 47 for ARMv8, but we use the descaddrmask
7625      * up to bit 39 for AArch32, because we don't need other bits in that case
7626      * to construct next descriptor address (anyway they should be all zeroes).
7627      */
7628     descaddrmask = ((1ull << (aarch64 ? 48 : 40)) - 1) &
7629                    ~indexmask_grainsize;
7630
7631     /* Secure accesses start with the page table in secure memory and
7632      * can be downgraded to non-secure at any step. Non-secure accesses
7633      * remain non-secure. We implement this by just ORing in the NSTable/NS
7634      * bits at each step.
7635      */
7636     tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4);
7637     for (;;) {
7638         uint64_t descriptor;
7639         bool nstable;
7640
7641         descaddr |= (address >> (stride * (4 - level))) & indexmask;
7642         descaddr &= ~7ULL;
7643         nstable = extract32(tableattrs, 4, 1);
7644         descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fsr, fi);
7645         if (fi->s1ptw) {
7646             goto do_fault;
7647         }
7648
7649         if (!(descriptor & 1) ||
7650             (!(descriptor & 2) && (level == 3))) {
7651             /* Invalid, or the Reserved level 3 encoding */
7652             goto do_fault;
7653         }
7654         descaddr = descriptor & descaddrmask;
7655
7656         if ((descriptor & 2) && (level < 3)) {
7657             /* Table entry. The top five bits are attributes which  may
7658              * propagate down through lower levels of the table (and
7659              * which are all arranged so that 0 means "no effect", so
7660              * we can gather them up by ORing in the bits at each level).
7661              */
7662             tableattrs |= extract64(descriptor, 59, 5);
7663             level++;
7664             indexmask = indexmask_grainsize;
7665             continue;
7666         }
7667         /* Block entry at level 1 or 2, or page entry at level 3.
7668          * These are basically the same thing, although the number
7669          * of bits we pull in from the vaddr varies.
7670          */
7671         page_size = (1ULL << ((stride * (4 - level)) + 3));
7672         descaddr |= (address & (page_size - 1));
7673         /* Extract attributes from the descriptor */
7674         attrs = extract64(descriptor, 2, 10)
7675             | (extract64(descriptor, 52, 12) << 10);
7676
7677         if (mmu_idx == ARMMMUIdx_S2NS) {
7678             /* Stage 2 table descriptors do not include any attribute fields */
7679             break;
7680         }
7681         /* Merge in attributes from table descriptors */
7682         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
7683         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
7684         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
7685          * means "force PL1 access only", which means forcing AP[1] to 0.
7686          */
7687         if (extract32(tableattrs, 2, 1)) {
7688             attrs &= ~(1 << 4);
7689         }
7690         attrs |= nstable << 3; /* NS */
7691         break;
7692     }
7693     /* Here descaddr is the final physical address, and attributes
7694      * are all in attrs.
7695      */
7696     fault_type = access_fault;
7697     if ((attrs & (1 << 8)) == 0) {
7698         /* Access flag */
7699         goto do_fault;
7700     }
7701
7702     ap = extract32(attrs, 4, 2);
7703     xn = extract32(attrs, 12, 1);
7704
7705     if (mmu_idx == ARMMMUIdx_S2NS) {
7706         ns = true;
7707         *prot = get_S2prot(env, ap, xn);
7708     } else {
7709         ns = extract32(attrs, 3, 1);
7710         pxn = extract32(attrs, 11, 1);
7711         *prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn);
7712     }
7713
7714     fault_type = permission_fault;
7715     if (!(*prot & (1 << access_type))) {
7716         goto do_fault;
7717     }
7718
7719     if (ns) {
7720         /* The NS bit will (as required by the architecture) have no effect if
7721          * the CPU doesn't support TZ or this is a non-secure translation
7722          * regime, because the attribute will already be non-secure.
7723          */
7724         txattrs->secure = false;
7725     }
7726     *phys_ptr = descaddr;
7727     *page_size_ptr = page_size;
7728     return false;
7729
7730 do_fault:
7731     /* Long-descriptor format IFSR/DFSR value */
7732     *fsr = (1 << 9) | (fault_type << 2) | level;
7733     /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2.  */
7734     fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_S2NS);
7735     return true;
7736 }
7737
7738 static inline void get_phys_addr_pmsav7_default(CPUARMState *env,
7739                                                 ARMMMUIdx mmu_idx,
7740                                                 int32_t address, int *prot)
7741 {
7742     *prot = PAGE_READ | PAGE_WRITE;
7743     switch (address) {
7744     case 0xF0000000 ... 0xFFFFFFFF:
7745         if (regime_sctlr(env, mmu_idx) & SCTLR_V) { /* hivecs execing is ok */
7746             *prot |= PAGE_EXEC;
7747         }
7748         break;
7749     case 0x00000000 ... 0x7FFFFFFF:
7750         *prot |= PAGE_EXEC;
7751         break;
7752     }
7753
7754 }
7755
7756 static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address,
7757                                  int access_type, ARMMMUIdx mmu_idx,
7758                                  hwaddr *phys_ptr, int *prot, uint32_t *fsr)
7759 {
7760     ARMCPU *cpu = arm_env_get_cpu(env);
7761     int n;
7762     bool is_user = regime_is_user(env, mmu_idx);
7763
7764     *phys_ptr = address;
7765     *prot = 0;
7766
7767     if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */
7768         get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
7769     } else { /* MPU enabled */
7770         for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) {
7771             /* region search */
7772             uint32_t base = env->pmsav7.drbar[n];
7773             uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5);
7774             uint32_t rmask;
7775             bool srdis = false;
7776
7777             if (!(env->pmsav7.drsr[n] & 0x1)) {
7778                 continue;
7779             }
7780
7781             if (!rsize) {
7782                 qemu_log_mask(LOG_GUEST_ERROR, "DRSR.Rsize field can not be 0");
7783                 continue;
7784             }
7785             rsize++;
7786             rmask = (1ull << rsize) - 1;
7787
7788             if (base & rmask) {
7789                 qemu_log_mask(LOG_GUEST_ERROR, "DRBAR %" PRIx32 " misaligned "
7790                               "to DRSR region size, mask = %" PRIx32,
7791                               base, rmask);
7792                 continue;
7793             }
7794
7795             if (address < base || address > base + rmask) {
7796                 continue;
7797             }
7798
7799             /* Region matched */
7800
7801             if (rsize >= 8) { /* no subregions for regions < 256 bytes */
7802                 int i, snd;
7803                 uint32_t srdis_mask;
7804
7805                 rsize -= 3; /* sub region size (power of 2) */
7806                 snd = ((address - base) >> rsize) & 0x7;
7807                 srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1);
7808
7809                 srdis_mask = srdis ? 0x3 : 0x0;
7810                 for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) {
7811                     /* This will check in groups of 2, 4 and then 8, whether
7812                      * the subregion bits are consistent. rsize is incremented
7813                      * back up to give the region size, considering consistent
7814                      * adjacent subregions as one region. Stop testing if rsize
7815                      * is already big enough for an entire QEMU page.
7816                      */
7817                     int snd_rounded = snd & ~(i - 1);
7818                     uint32_t srdis_multi = extract32(env->pmsav7.drsr[n],
7819                                                      snd_rounded + 8, i);
7820                     if (srdis_mask ^ srdis_multi) {
7821                         break;
7822                     }
7823                     srdis_mask = (srdis_mask << i) | srdis_mask;
7824                     rsize++;
7825                 }
7826             }
7827             if (rsize < TARGET_PAGE_BITS) {
7828                 qemu_log_mask(LOG_UNIMP, "No support for MPU (sub)region"
7829                               "alignment of %" PRIu32 " bits. Minimum is %d\n",
7830                               rsize, TARGET_PAGE_BITS);
7831                 continue;
7832             }
7833             if (srdis) {
7834                 continue;
7835             }
7836             break;
7837         }
7838
7839         if (n == -1) { /* no hits */
7840             if (cpu->pmsav7_dregion &&
7841                 (is_user || !(regime_sctlr(env, mmu_idx) & SCTLR_BR))) {
7842                 /* background fault */
7843                 *fsr = 0;
7844                 return true;
7845             }
7846             get_phys_addr_pmsav7_default(env, mmu_idx, address, prot);
7847         } else { /* a MPU hit! */
7848             uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3);
7849
7850             if (is_user) { /* User mode AP bit decoding */
7851                 switch (ap) {
7852                 case 0:
7853                 case 1:
7854                 case 5:
7855                     break; /* no access */
7856                 case 3:
7857                     *prot |= PAGE_WRITE;
7858                     /* fall through */
7859                 case 2:
7860                 case 6:
7861                     *prot |= PAGE_READ | PAGE_EXEC;
7862                     break;
7863                 default:
7864                     qemu_log_mask(LOG_GUEST_ERROR,
7865                                   "Bad value for AP bits in DRACR %"
7866                                   PRIx32 "\n", ap);
7867                 }
7868             } else { /* Priv. mode AP bits decoding */
7869                 switch (ap) {
7870                 case 0:
7871                     break; /* no access */
7872                 case 1:
7873                 case 2:
7874                 case 3:
7875                     *prot |= PAGE_WRITE;
7876                     /* fall through */
7877                 case 5:
7878                 case 6:
7879                     *prot |= PAGE_READ | PAGE_EXEC;
7880                     break;
7881                 default:
7882                     qemu_log_mask(LOG_GUEST_ERROR,
7883                                   "Bad value for AP bits in DRACR %"
7884                                   PRIx32 "\n", ap);
7885                 }
7886             }
7887
7888             /* execute never */
7889             if (env->pmsav7.dracr[n] & (1 << 12)) {
7890                 *prot &= ~PAGE_EXEC;
7891             }
7892         }
7893     }
7894
7895     *fsr = 0x00d; /* Permission fault */
7896     return !(*prot & (1 << access_type));
7897 }
7898
7899 static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address,
7900                                  int access_type, ARMMMUIdx mmu_idx,
7901                                  hwaddr *phys_ptr, int *prot, uint32_t *fsr)
7902 {
7903     int n;
7904     uint32_t mask;
7905     uint32_t base;
7906     bool is_user = regime_is_user(env, mmu_idx);
7907
7908     *phys_ptr = address;
7909     for (n = 7; n >= 0; n--) {
7910         base = env->cp15.c6_region[n];
7911         if ((base & 1) == 0) {
7912             continue;
7913         }
7914         mask = 1 << ((base >> 1) & 0x1f);
7915         /* Keep this shift separate from the above to avoid an
7916            (undefined) << 32.  */
7917         mask = (mask << 1) - 1;
7918         if (((base ^ address) & ~mask) == 0) {
7919             break;
7920         }
7921     }
7922     if (n < 0) {
7923         *fsr = 2;
7924         return true;
7925     }
7926
7927     if (access_type == 2) {
7928         mask = env->cp15.pmsav5_insn_ap;
7929     } else {
7930         mask = env->cp15.pmsav5_data_ap;
7931     }
7932     mask = (mask >> (n * 4)) & 0xf;
7933     switch (mask) {
7934     case 0:
7935         *fsr = 1;
7936         return true;
7937     case 1:
7938         if (is_user) {
7939             *fsr = 1;
7940             return true;
7941         }
7942         *prot = PAGE_READ | PAGE_WRITE;
7943         break;
7944     case 2:
7945         *prot = PAGE_READ;
7946         if (!is_user) {
7947             *prot |= PAGE_WRITE;
7948         }
7949         break;
7950     case 3:
7951         *prot = PAGE_READ | PAGE_WRITE;
7952         break;
7953     case 5:
7954         if (is_user) {
7955             *fsr = 1;
7956             return true;
7957         }
7958         *prot = PAGE_READ;
7959         break;
7960     case 6:
7961         *prot = PAGE_READ;
7962         break;
7963     default:
7964         /* Bad permission.  */
7965         *fsr = 1;
7966         return true;
7967     }
7968     *prot |= PAGE_EXEC;
7969     return false;
7970 }
7971
7972 /* get_phys_addr - get the physical address for this virtual address
7973  *
7974  * Find the physical address corresponding to the given virtual address,
7975  * by doing a translation table walk on MMU based systems or using the
7976  * MPU state on MPU based systems.
7977  *
7978  * Returns false if the translation was successful. Otherwise, phys_ptr, attrs,
7979  * prot and page_size may not be filled in, and the populated fsr value provides
7980  * information on why the translation aborted, in the format of a
7981  * DFSR/IFSR fault register, with the following caveats:
7982  *  * we honour the short vs long DFSR format differences.
7983  *  * the WnR bit is never set (the caller must do this).
7984  *  * for PSMAv5 based systems we don't bother to return a full FSR format
7985  *    value.
7986  *
7987  * @env: CPUARMState
7988  * @address: virtual address to get physical address for
7989  * @access_type: 0 for read, 1 for write, 2 for execute
7990  * @mmu_idx: MMU index indicating required translation regime
7991  * @phys_ptr: set to the physical address corresponding to the virtual address
7992  * @attrs: set to the memory transaction attributes to use
7993  * @prot: set to the permissions for the page containing phys_ptr
7994  * @page_size: set to the size of the page containing phys_ptr
7995  * @fsr: set to the DFSR/IFSR value on failure
7996  */
7997 static bool get_phys_addr(CPUARMState *env, target_ulong address,
7998                           int access_type, ARMMMUIdx mmu_idx,
7999                           hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot,
8000                           target_ulong *page_size, uint32_t *fsr,
8001                           ARMMMUFaultInfo *fi)
8002 {
8003     if (mmu_idx == ARMMMUIdx_S12NSE0 || mmu_idx == ARMMMUIdx_S12NSE1) {
8004         /* Call ourselves recursively to do the stage 1 and then stage 2
8005          * translations.
8006          */
8007         if (arm_feature(env, ARM_FEATURE_EL2)) {
8008             hwaddr ipa;
8009             int s2_prot;
8010             int ret;
8011
8012             ret = get_phys_addr(env, address, access_type,
8013                                 mmu_idx + ARMMMUIdx_S1NSE0, &ipa, attrs,
8014                                 prot, page_size, fsr, fi);
8015
8016             /* If S1 fails or S2 is disabled, return early.  */
8017             if (ret || regime_translation_disabled(env, ARMMMUIdx_S2NS)) {
8018                 *phys_ptr = ipa;
8019                 return ret;
8020             }
8021
8022             /* S1 is done. Now do S2 translation.  */
8023             ret = get_phys_addr_lpae(env, ipa, access_type, ARMMMUIdx_S2NS,
8024                                      phys_ptr, attrs, &s2_prot,
8025                                      page_size, fsr, fi);
8026             fi->s2addr = ipa;
8027             /* Combine the S1 and S2 perms.  */
8028             *prot &= s2_prot;
8029             return ret;
8030         } else {
8031             /*
8032              * For non-EL2 CPUs a stage1+stage2 translation is just stage 1.
8033              */
8034             mmu_idx += ARMMMUIdx_S1NSE0;
8035         }
8036     }
8037
8038     /* The page table entries may downgrade secure to non-secure, but
8039      * cannot upgrade an non-secure translation regime's attributes
8040      * to secure.
8041      */
8042     attrs->secure = regime_is_secure(env, mmu_idx);
8043     attrs->user = regime_is_user(env, mmu_idx);
8044
8045     /* Fast Context Switch Extension. This doesn't exist at all in v8.
8046      * In v7 and earlier it affects all stage 1 translations.
8047      */
8048     if (address < 0x02000000 && mmu_idx != ARMMMUIdx_S2NS
8049         && !arm_feature(env, ARM_FEATURE_V8)) {
8050         if (regime_el(env, mmu_idx) == 3) {
8051             address += env->cp15.fcseidr_s;
8052         } else {
8053             address += env->cp15.fcseidr_ns;
8054         }
8055     }
8056
8057     /* pmsav7 has special handling for when MPU is disabled so call it before
8058      * the common MMU/MPU disabled check below.
8059      */
8060     if (arm_feature(env, ARM_FEATURE_MPU) &&
8061         arm_feature(env, ARM_FEATURE_V7)) {
8062         *page_size = TARGET_PAGE_SIZE;
8063         return get_phys_addr_pmsav7(env, address, access_type, mmu_idx,
8064                                     phys_ptr, prot, fsr);
8065     }
8066
8067     if (regime_translation_disabled(env, mmu_idx)) {
8068         /* MMU/MPU disabled.  */
8069         *phys_ptr = address;
8070         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
8071         *page_size = TARGET_PAGE_SIZE;
8072         return 0;
8073     }
8074
8075     if (arm_feature(env, ARM_FEATURE_MPU)) {
8076         /* Pre-v7 MPU */
8077         *page_size = TARGET_PAGE_SIZE;
8078         return get_phys_addr_pmsav5(env, address, access_type, mmu_idx,
8079                                     phys_ptr, prot, fsr);
8080     }
8081
8082     if (regime_using_lpae_format(env, mmu_idx)) {
8083         return get_phys_addr_lpae(env, address, access_type, mmu_idx, phys_ptr,
8084                                   attrs, prot, page_size, fsr, fi);
8085     } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) {
8086         return get_phys_addr_v6(env, address, access_type, mmu_idx, phys_ptr,
8087                                 attrs, prot, page_size, fsr, fi);
8088     } else {
8089         return get_phys_addr_v5(env, address, access_type, mmu_idx, phys_ptr,
8090                                 prot, page_size, fsr, fi);
8091     }
8092 }
8093
8094 /* Walk the page table and (if the mapping exists) add the page
8095  * to the TLB. Return false on success, or true on failure. Populate
8096  * fsr with ARM DFSR/IFSR fault register format value on failure.
8097  */
8098 bool arm_tlb_fill(CPUState *cs, vaddr address,
8099                   int access_type, int mmu_idx, uint32_t *fsr,
8100                   ARMMMUFaultInfo *fi)
8101 {
8102     ARMCPU *cpu = ARM_CPU(cs);
8103     CPUARMState *env = &cpu->env;
8104     hwaddr phys_addr;
8105     target_ulong page_size;
8106     int prot;
8107     int ret;
8108     MemTxAttrs attrs = {};
8109
8110     ret = get_phys_addr(env, address, access_type, mmu_idx, &phys_addr,
8111                         &attrs, &prot, &page_size, fsr, fi);
8112     if (!ret) {
8113         /* Map a single [sub]page.  */
8114         phys_addr &= TARGET_PAGE_MASK;
8115         address &= TARGET_PAGE_MASK;
8116         tlb_set_page_with_attrs(cs, address, phys_addr, attrs,
8117                                 prot, mmu_idx, page_size);
8118         return 0;
8119     }
8120
8121     return ret;
8122 }
8123
8124 hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr,
8125                                          MemTxAttrs *attrs)
8126 {
8127     ARMCPU *cpu = ARM_CPU(cs);
8128     CPUARMState *env = &cpu->env;
8129     hwaddr phys_addr;
8130     target_ulong page_size;
8131     int prot;
8132     bool ret;
8133     uint32_t fsr;
8134     ARMMMUFaultInfo fi = {};
8135
8136     *attrs = (MemTxAttrs) {};
8137
8138     ret = get_phys_addr(env, addr, 0, cpu_mmu_index(env, false), &phys_addr,
8139                         attrs, &prot, &page_size, &fsr, &fi);
8140
8141     if (ret) {
8142         return -1;
8143     }
8144     return phys_addr;
8145 }
8146
8147 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
8148 {
8149     ARMCPU *cpu = arm_env_get_cpu(env);
8150
8151     switch (reg) {
8152     case 0: /* APSR */
8153         return xpsr_read(env) & 0xf8000000;
8154     case 1: /* IAPSR */
8155         return xpsr_read(env) & 0xf80001ff;
8156     case 2: /* EAPSR */
8157         return xpsr_read(env) & 0xff00fc00;
8158     case 3: /* xPSR */
8159         return xpsr_read(env) & 0xff00fdff;
8160     case 5: /* IPSR */
8161         return xpsr_read(env) & 0x000001ff;
8162     case 6: /* EPSR */
8163         return xpsr_read(env) & 0x0700fc00;
8164     case 7: /* IEPSR */
8165         return xpsr_read(env) & 0x0700edff;
8166     case 8: /* MSP */
8167         return env->v7m.current_sp ? env->v7m.other_sp : env->regs[13];
8168     case 9: /* PSP */
8169         return env->v7m.current_sp ? env->regs[13] : env->v7m.other_sp;
8170     case 16: /* PRIMASK */
8171         return (env->daif & PSTATE_I) != 0;
8172     case 17: /* BASEPRI */
8173     case 18: /* BASEPRI_MAX */
8174         return env->v7m.basepri;
8175     case 19: /* FAULTMASK */
8176         return (env->daif & PSTATE_F) != 0;
8177     case 20: /* CONTROL */
8178         return env->v7m.control;
8179     default:
8180         /* ??? For debugging only.  */
8181         cpu_abort(CPU(cpu), "Unimplemented system register read (%d)\n", reg);
8182         return 0;
8183     }
8184 }
8185
8186 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
8187 {
8188     ARMCPU *cpu = arm_env_get_cpu(env);
8189
8190     switch (reg) {
8191     case 0: /* APSR */
8192         xpsr_write(env, val, 0xf8000000);
8193         break;
8194     case 1: /* IAPSR */
8195         xpsr_write(env, val, 0xf8000000);
8196         break;
8197     case 2: /* EAPSR */
8198         xpsr_write(env, val, 0xfe00fc00);
8199         break;
8200     case 3: /* xPSR */
8201         xpsr_write(env, val, 0xfe00fc00);
8202         break;
8203     case 5: /* IPSR */
8204         /* IPSR bits are readonly.  */
8205         break;
8206     case 6: /* EPSR */
8207         xpsr_write(env, val, 0x0600fc00);
8208         break;
8209     case 7: /* IEPSR */
8210         xpsr_write(env, val, 0x0600fc00);
8211         break;
8212     case 8: /* MSP */
8213         if (env->v7m.current_sp)
8214             env->v7m.other_sp = val;
8215         else
8216             env->regs[13] = val;
8217         break;
8218     case 9: /* PSP */
8219         if (env->v7m.current_sp)
8220             env->regs[13] = val;
8221         else
8222             env->v7m.other_sp = val;
8223         break;
8224     case 16: /* PRIMASK */
8225         if (val & 1) {
8226             env->daif |= PSTATE_I;
8227         } else {
8228             env->daif &= ~PSTATE_I;
8229         }
8230         break;
8231     case 17: /* BASEPRI */
8232         env->v7m.basepri = val & 0xff;
8233         break;
8234     case 18: /* BASEPRI_MAX */
8235         val &= 0xff;
8236         if (val != 0 && (val < env->v7m.basepri || env->v7m.basepri == 0))
8237             env->v7m.basepri = val;
8238         break;
8239     case 19: /* FAULTMASK */
8240         if (val & 1) {
8241             env->daif |= PSTATE_F;
8242         } else {
8243             env->daif &= ~PSTATE_F;
8244         }
8245         break;
8246     case 20: /* CONTROL */
8247         env->v7m.control = val & 3;
8248         switch_v7m_sp(env, (val & 2) != 0);
8249         break;
8250     default:
8251         /* ??? For debugging only.  */
8252         cpu_abort(CPU(cpu), "Unimplemented system register write (%d)\n", reg);
8253         return;
8254     }
8255 }
8256
8257 #endif
8258
8259 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
8260 {
8261     /* Implement DC ZVA, which zeroes a fixed-length block of memory.
8262      * Note that we do not implement the (architecturally mandated)
8263      * alignment fault for attempts to use this on Device memory
8264      * (which matches the usual QEMU behaviour of not implementing either
8265      * alignment faults or any memory attribute handling).
8266      */
8267
8268     ARMCPU *cpu = arm_env_get_cpu(env);
8269     uint64_t blocklen = 4 << cpu->dcz_blocksize;
8270     uint64_t vaddr = vaddr_in & ~(blocklen - 1);
8271
8272 #ifndef CONFIG_USER_ONLY
8273     {
8274         /* Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than
8275          * the block size so we might have to do more than one TLB lookup.
8276          * We know that in fact for any v8 CPU the page size is at least 4K
8277          * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only
8278          * 1K as an artefact of legacy v5 subpage support being present in the
8279          * same QEMU executable.
8280          */
8281         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
8282         void *hostaddr[maxidx];
8283         int try, i;
8284         unsigned mmu_idx = cpu_mmu_index(env, false);
8285         TCGMemOpIdx oi = make_memop_idx(MO_UB, mmu_idx);
8286
8287         for (try = 0; try < 2; try++) {
8288
8289             for (i = 0; i < maxidx; i++) {
8290                 hostaddr[i] = tlb_vaddr_to_host(env,
8291                                                 vaddr + TARGET_PAGE_SIZE * i,
8292                                                 1, mmu_idx);
8293                 if (!hostaddr[i]) {
8294                     break;
8295                 }
8296             }
8297             if (i == maxidx) {
8298                 /* If it's all in the TLB it's fair game for just writing to;
8299                  * we know we don't need to update dirty status, etc.
8300                  */
8301                 for (i = 0; i < maxidx - 1; i++) {
8302                     memset(hostaddr[i], 0, TARGET_PAGE_SIZE);
8303                 }
8304                 memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE));
8305                 return;
8306             }
8307             /* OK, try a store and see if we can populate the tlb. This
8308              * might cause an exception if the memory isn't writable,
8309              * in which case we will longjmp out of here. We must for
8310              * this purpose use the actual register value passed to us
8311              * so that we get the fault address right.
8312              */
8313             helper_ret_stb_mmu(env, vaddr_in, 0, oi, GETPC());
8314             /* Now we can populate the other TLB entries, if any */
8315             for (i = 0; i < maxidx; i++) {
8316                 uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
8317                 if (va != (vaddr_in & TARGET_PAGE_MASK)) {
8318                     helper_ret_stb_mmu(env, va, 0, oi, GETPC());
8319                 }
8320             }
8321         }
8322
8323         /* Slow path (probably attempt to do this to an I/O device or
8324          * similar, or clearing of a block of code we have translations
8325          * cached for). Just do a series of byte writes as the architecture
8326          * demands. It's not worth trying to use a cpu_physical_memory_map(),
8327          * memset(), unmap() sequence here because:
8328          *  + we'd need to account for the blocksize being larger than a page
8329          *  + the direct-RAM access case is almost always going to be dealt
8330          *    with in the fastpath code above, so there's no speed benefit
8331          *  + we would have to deal with the map returning NULL because the
8332          *    bounce buffer was in use
8333          */
8334         for (i = 0; i < blocklen; i++) {
8335             helper_ret_stb_mmu(env, vaddr + i, 0, oi, GETPC());
8336         }
8337     }
8338 #else
8339     memset(g2h(vaddr), 0, blocklen);
8340 #endif
8341 }
8342
8343 /* Note that signed overflow is undefined in C.  The following routines are
8344    careful to use unsigned types where modulo arithmetic is required.
8345    Failure to do so _will_ break on newer gcc.  */
8346
8347 /* Signed saturating arithmetic.  */
8348
8349 /* Perform 16-bit signed saturating addition.  */
8350 static inline uint16_t add16_sat(uint16_t a, uint16_t b)
8351 {
8352     uint16_t res;
8353
8354     res = a + b;
8355     if (((res ^ a) & 0x8000) && !((a ^ b) & 0x8000)) {
8356         if (a & 0x8000)
8357             res = 0x8000;
8358         else
8359             res = 0x7fff;
8360     }
8361     return res;
8362 }
8363
8364 /* Perform 8-bit signed saturating addition.  */
8365 static inline uint8_t add8_sat(uint8_t a, uint8_t b)
8366 {
8367     uint8_t res;
8368
8369     res = a + b;
8370     if (((res ^ a) & 0x80) && !((a ^ b) & 0x80)) {
8371         if (a & 0x80)
8372             res = 0x80;
8373         else
8374             res = 0x7f;
8375     }
8376     return res;
8377 }
8378
8379 /* Perform 16-bit signed saturating subtraction.  */
8380 static inline uint16_t sub16_sat(uint16_t a, uint16_t b)
8381 {
8382     uint16_t res;
8383
8384     res = a - b;
8385     if (((res ^ a) & 0x8000) && ((a ^ b) & 0x8000)) {
8386         if (a & 0x8000)
8387             res = 0x8000;
8388         else
8389             res = 0x7fff;
8390     }
8391     return res;
8392 }
8393
8394 /* Perform 8-bit signed saturating subtraction.  */
8395 static inline uint8_t sub8_sat(uint8_t a, uint8_t b)
8396 {
8397     uint8_t res;
8398
8399     res = a - b;
8400     if (((res ^ a) & 0x80) && ((a ^ b) & 0x80)) {
8401         if (a & 0x80)
8402             res = 0x80;
8403         else
8404             res = 0x7f;
8405     }
8406     return res;
8407 }
8408
8409 #define ADD16(a, b, n) RESULT(add16_sat(a, b), n, 16);
8410 #define SUB16(a, b, n) RESULT(sub16_sat(a, b), n, 16);
8411 #define ADD8(a, b, n)  RESULT(add8_sat(a, b), n, 8);
8412 #define SUB8(a, b, n)  RESULT(sub8_sat(a, b), n, 8);
8413 #define PFX q
8414
8415 #include "op_addsub.h"
8416
8417 /* Unsigned saturating arithmetic.  */
8418 static inline uint16_t add16_usat(uint16_t a, uint16_t b)
8419 {
8420     uint16_t res;
8421     res = a + b;
8422     if (res < a)
8423         res = 0xffff;
8424     return res;
8425 }
8426
8427 static inline uint16_t sub16_usat(uint16_t a, uint16_t b)
8428 {
8429     if (a > b)
8430         return a - b;
8431     else
8432         return 0;
8433 }
8434
8435 static inline uint8_t add8_usat(uint8_t a, uint8_t b)
8436 {
8437     uint8_t res;
8438     res = a + b;
8439     if (res < a)
8440         res = 0xff;
8441     return res;
8442 }
8443
8444 static inline uint8_t sub8_usat(uint8_t a, uint8_t b)
8445 {
8446     if (a > b)
8447         return a - b;
8448     else
8449         return 0;
8450 }
8451
8452 #define ADD16(a, b, n) RESULT(add16_usat(a, b), n, 16);
8453 #define SUB16(a, b, n) RESULT(sub16_usat(a, b), n, 16);
8454 #define ADD8(a, b, n)  RESULT(add8_usat(a, b), n, 8);
8455 #define SUB8(a, b, n)  RESULT(sub8_usat(a, b), n, 8);
8456 #define PFX uq
8457
8458 #include "op_addsub.h"
8459
8460 /* Signed modulo arithmetic.  */
8461 #define SARITH16(a, b, n, op) do { \
8462     int32_t sum; \
8463     sum = (int32_t)(int16_t)(a) op (int32_t)(int16_t)(b); \
8464     RESULT(sum, n, 16); \
8465     if (sum >= 0) \
8466         ge |= 3 << (n * 2); \
8467     } while(0)
8468
8469 #define SARITH8(a, b, n, op) do { \
8470     int32_t sum; \
8471     sum = (int32_t)(int8_t)(a) op (int32_t)(int8_t)(b); \
8472     RESULT(sum, n, 8); \
8473     if (sum >= 0) \
8474         ge |= 1 << n; \
8475     } while(0)
8476
8477
8478 #define ADD16(a, b, n) SARITH16(a, b, n, +)
8479 #define SUB16(a, b, n) SARITH16(a, b, n, -)
8480 #define ADD8(a, b, n)  SARITH8(a, b, n, +)
8481 #define SUB8(a, b, n)  SARITH8(a, b, n, -)
8482 #define PFX s
8483 #define ARITH_GE
8484
8485 #include "op_addsub.h"
8486
8487 /* Unsigned modulo arithmetic.  */
8488 #define ADD16(a, b, n) do { \
8489     uint32_t sum; \
8490     sum = (uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b); \
8491     RESULT(sum, n, 16); \
8492     if ((sum >> 16) == 1) \
8493         ge |= 3 << (n * 2); \
8494     } while(0)
8495
8496 #define ADD8(a, b, n) do { \
8497     uint32_t sum; \
8498     sum = (uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b); \
8499     RESULT(sum, n, 8); \
8500     if ((sum >> 8) == 1) \
8501         ge |= 1 << n; \
8502     } while(0)
8503
8504 #define SUB16(a, b, n) do { \
8505     uint32_t sum; \
8506     sum = (uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b); \
8507     RESULT(sum, n, 16); \
8508     if ((sum >> 16) == 0) \
8509         ge |= 3 << (n * 2); \
8510     } while(0)
8511
8512 #define SUB8(a, b, n) do { \
8513     uint32_t sum; \
8514     sum = (uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b); \
8515     RESULT(sum, n, 8); \
8516     if ((sum >> 8) == 0) \
8517         ge |= 1 << n; \
8518     } while(0)
8519
8520 #define PFX u
8521 #define ARITH_GE
8522
8523 #include "op_addsub.h"
8524
8525 /* Halved signed arithmetic.  */
8526 #define ADD16(a, b, n) \
8527   RESULT(((int32_t)(int16_t)(a) + (int32_t)(int16_t)(b)) >> 1, n, 16)
8528 #define SUB16(a, b, n) \
8529   RESULT(((int32_t)(int16_t)(a) - (int32_t)(int16_t)(b)) >> 1, n, 16)
8530 #define ADD8(a, b, n) \
8531   RESULT(((int32_t)(int8_t)(a) + (int32_t)(int8_t)(b)) >> 1, n, 8)
8532 #define SUB8(a, b, n) \
8533   RESULT(((int32_t)(int8_t)(a) - (int32_t)(int8_t)(b)) >> 1, n, 8)
8534 #define PFX sh
8535
8536 #include "op_addsub.h"
8537
8538 /* Halved unsigned arithmetic.  */
8539 #define ADD16(a, b, n) \
8540   RESULT(((uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b)) >> 1, n, 16)
8541 #define SUB16(a, b, n) \
8542   RESULT(((uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b)) >> 1, n, 16)
8543 #define ADD8(a, b, n) \
8544   RESULT(((uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b)) >> 1, n, 8)
8545 #define SUB8(a, b, n) \
8546   RESULT(((uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b)) >> 1, n, 8)
8547 #define PFX uh
8548
8549 #include "op_addsub.h"
8550
8551 static inline uint8_t do_usad(uint8_t a, uint8_t b)
8552 {
8553     if (a > b)
8554         return a - b;
8555     else
8556         return b - a;
8557 }
8558
8559 /* Unsigned sum of absolute byte differences.  */
8560 uint32_t HELPER(usad8)(uint32_t a, uint32_t b)
8561 {
8562     uint32_t sum;
8563     sum = do_usad(a, b);
8564     sum += do_usad(a >> 8, b >> 8);
8565     sum += do_usad(a >> 16, b >>16);
8566     sum += do_usad(a >> 24, b >> 24);
8567     return sum;
8568 }
8569
8570 /* For ARMv6 SEL instruction.  */
8571 uint32_t HELPER(sel_flags)(uint32_t flags, uint32_t a, uint32_t b)
8572 {
8573     uint32_t mask;
8574
8575     mask = 0;
8576     if (flags & 1)
8577         mask |= 0xff;
8578     if (flags & 2)
8579         mask |= 0xff00;
8580     if (flags & 4)
8581         mask |= 0xff0000;
8582     if (flags & 8)
8583         mask |= 0xff000000;
8584     return (a & mask) | (b & ~mask);
8585 }
8586
8587 /* VFP support.  We follow the convention used for VFP instructions:
8588    Single precision routines have a "s" suffix, double precision a
8589    "d" suffix.  */
8590
8591 /* Convert host exception flags to vfp form.  */
8592 static inline int vfp_exceptbits_from_host(int host_bits)
8593 {
8594     int target_bits = 0;
8595
8596     if (host_bits & float_flag_invalid)
8597         target_bits |= 1;
8598     if (host_bits & float_flag_divbyzero)
8599         target_bits |= 2;
8600     if (host_bits & float_flag_overflow)
8601         target_bits |= 4;
8602     if (host_bits & (float_flag_underflow | float_flag_output_denormal))
8603         target_bits |= 8;
8604     if (host_bits & float_flag_inexact)
8605         target_bits |= 0x10;
8606     if (host_bits & float_flag_input_denormal)
8607         target_bits |= 0x80;
8608     return target_bits;
8609 }
8610
8611 uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env)
8612 {
8613     int i;
8614     uint32_t fpscr;
8615
8616     fpscr = (env->vfp.xregs[ARM_VFP_FPSCR] & 0xffc8ffff)
8617             | (env->vfp.vec_len << 16)
8618             | (env->vfp.vec_stride << 20);
8619     i = get_float_exception_flags(&env->vfp.fp_status);
8620     i |= get_float_exception_flags(&env->vfp.standard_fp_status);
8621     fpscr |= vfp_exceptbits_from_host(i);
8622     return fpscr;
8623 }
8624
8625 uint32_t vfp_get_fpscr(CPUARMState *env)
8626 {
8627     return HELPER(vfp_get_fpscr)(env);
8628 }
8629
8630 /* Convert vfp exception flags to target form.  */
8631 static inline int vfp_exceptbits_to_host(int target_bits)
8632 {
8633     int host_bits = 0;
8634
8635     if (target_bits & 1)
8636         host_bits |= float_flag_invalid;
8637     if (target_bits & 2)
8638         host_bits |= float_flag_divbyzero;
8639     if (target_bits & 4)
8640         host_bits |= float_flag_overflow;
8641     if (target_bits & 8)
8642         host_bits |= float_flag_underflow;
8643     if (target_bits & 0x10)
8644         host_bits |= float_flag_inexact;
8645     if (target_bits & 0x80)
8646         host_bits |= float_flag_input_denormal;
8647     return host_bits;
8648 }
8649
8650 void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val)
8651 {
8652     int i;
8653     uint32_t changed;
8654
8655     changed = env->vfp.xregs[ARM_VFP_FPSCR];
8656     env->vfp.xregs[ARM_VFP_FPSCR] = (val & 0xffc8ffff);
8657     env->vfp.vec_len = (val >> 16) & 7;
8658     env->vfp.vec_stride = (val >> 20) & 3;
8659
8660     changed ^= val;
8661     if (changed & (3 << 22)) {
8662         i = (val >> 22) & 3;
8663         switch (i) {
8664         case FPROUNDING_TIEEVEN:
8665             i = float_round_nearest_even;
8666             break;
8667         case FPROUNDING_POSINF:
8668             i = float_round_up;
8669             break;
8670         case FPROUNDING_NEGINF:
8671             i = float_round_down;
8672             break;
8673         case FPROUNDING_ZERO:
8674             i = float_round_to_zero;
8675             break;
8676         }
8677         set_float_rounding_mode(i, &env->vfp.fp_status);
8678     }
8679     if (changed & (1 << 24)) {
8680         set_flush_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
8681         set_flush_inputs_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
8682     }
8683     if (changed & (1 << 25))
8684         set_default_nan_mode((val & (1 << 25)) != 0, &env->vfp.fp_status);
8685
8686     i = vfp_exceptbits_to_host(val);
8687     set_float_exception_flags(i, &env->vfp.fp_status);
8688     set_float_exception_flags(0, &env->vfp.standard_fp_status);
8689 }
8690
8691 void vfp_set_fpscr(CPUARMState *env, uint32_t val)
8692 {
8693     HELPER(vfp_set_fpscr)(env, val);
8694 }
8695
8696 #define VFP_HELPER(name, p) HELPER(glue(glue(vfp_,name),p))
8697
8698 #define VFP_BINOP(name) \
8699 float32 VFP_HELPER(name, s)(float32 a, float32 b, void *fpstp) \
8700 { \
8701     float_status *fpst = fpstp; \
8702     return float32_ ## name(a, b, fpst); \
8703 } \
8704 float64 VFP_HELPER(name, d)(float64 a, float64 b, void *fpstp) \
8705 { \
8706     float_status *fpst = fpstp; \
8707     return float64_ ## name(a, b, fpst); \
8708 }
8709 VFP_BINOP(add)
8710 VFP_BINOP(sub)
8711 VFP_BINOP(mul)
8712 VFP_BINOP(div)
8713 VFP_BINOP(min)
8714 VFP_BINOP(max)
8715 VFP_BINOP(minnum)
8716 VFP_BINOP(maxnum)
8717 #undef VFP_BINOP
8718
8719 float32 VFP_HELPER(neg, s)(float32 a)
8720 {
8721     return float32_chs(a);
8722 }
8723
8724 float64 VFP_HELPER(neg, d)(float64 a)
8725 {
8726     return float64_chs(a);
8727 }
8728
8729 float32 VFP_HELPER(abs, s)(float32 a)
8730 {
8731     return float32_abs(a);
8732 }
8733
8734 float64 VFP_HELPER(abs, d)(float64 a)
8735 {
8736     return float64_abs(a);
8737 }
8738
8739 float32 VFP_HELPER(sqrt, s)(float32 a, CPUARMState *env)
8740 {
8741     return float32_sqrt(a, &env->vfp.fp_status);
8742 }
8743
8744 float64 VFP_HELPER(sqrt, d)(float64 a, CPUARMState *env)
8745 {
8746     return float64_sqrt(a, &env->vfp.fp_status);
8747 }
8748
8749 /* XXX: check quiet/signaling case */
8750 #define DO_VFP_cmp(p, type) \
8751 void VFP_HELPER(cmp, p)(type a, type b, CPUARMState *env)  \
8752 { \
8753     uint32_t flags; \
8754     switch(type ## _compare_quiet(a, b, &env->vfp.fp_status)) { \
8755     case 0: flags = 0x6; break; \
8756     case -1: flags = 0x8; break; \
8757     case 1: flags = 0x2; break; \
8758     default: case 2: flags = 0x3; break; \
8759     } \
8760     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
8761         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
8762 } \
8763 void VFP_HELPER(cmpe, p)(type a, type b, CPUARMState *env) \
8764 { \
8765     uint32_t flags; \
8766     switch(type ## _compare(a, b, &env->vfp.fp_status)) { \
8767     case 0: flags = 0x6; break; \
8768     case -1: flags = 0x8; break; \
8769     case 1: flags = 0x2; break; \
8770     default: case 2: flags = 0x3; break; \
8771     } \
8772     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
8773         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
8774 }
8775 DO_VFP_cmp(s, float32)
8776 DO_VFP_cmp(d, float64)
8777 #undef DO_VFP_cmp
8778
8779 /* Integer to float and float to integer conversions */
8780
8781 #define CONV_ITOF(name, fsz, sign) \
8782     float##fsz HELPER(name)(uint32_t x, void *fpstp) \
8783 { \
8784     float_status *fpst = fpstp; \
8785     return sign##int32_to_##float##fsz((sign##int32_t)x, fpst); \
8786 }
8787
8788 #define CONV_FTOI(name, fsz, sign, round) \
8789 uint32_t HELPER(name)(float##fsz x, void *fpstp) \
8790 { \
8791     float_status *fpst = fpstp; \
8792     if (float##fsz##_is_any_nan(x)) { \
8793         float_raise(float_flag_invalid, fpst); \
8794         return 0; \
8795     } \
8796     return float##fsz##_to_##sign##int32##round(x, fpst); \
8797 }
8798
8799 #define FLOAT_CONVS(name, p, fsz, sign) \
8800 CONV_ITOF(vfp_##name##to##p, fsz, sign) \
8801 CONV_FTOI(vfp_to##name##p, fsz, sign, ) \
8802 CONV_FTOI(vfp_to##name##z##p, fsz, sign, _round_to_zero)
8803
8804 FLOAT_CONVS(si, s, 32, )
8805 FLOAT_CONVS(si, d, 64, )
8806 FLOAT_CONVS(ui, s, 32, u)
8807 FLOAT_CONVS(ui, d, 64, u)
8808
8809 #undef CONV_ITOF
8810 #undef CONV_FTOI
8811 #undef FLOAT_CONVS
8812
8813 /* floating point conversion */
8814 float64 VFP_HELPER(fcvtd, s)(float32 x, CPUARMState *env)
8815 {
8816     float64 r = float32_to_float64(x, &env->vfp.fp_status);
8817     /* ARM requires that S<->D conversion of any kind of NaN generates
8818      * a quiet NaN by forcing the most significant frac bit to 1.
8819      */
8820     return float64_maybe_silence_nan(r, &env->vfp.fp_status);
8821 }
8822
8823 float32 VFP_HELPER(fcvts, d)(float64 x, CPUARMState *env)
8824 {
8825     float32 r =  float64_to_float32(x, &env->vfp.fp_status);
8826     /* ARM requires that S<->D conversion of any kind of NaN generates
8827      * a quiet NaN by forcing the most significant frac bit to 1.
8828      */
8829     return float32_maybe_silence_nan(r, &env->vfp.fp_status);
8830 }
8831
8832 /* VFP3 fixed point conversion.  */
8833 #define VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype) \
8834 float##fsz HELPER(vfp_##name##to##p)(uint##isz##_t  x, uint32_t shift, \
8835                                      void *fpstp) \
8836 { \
8837     float_status *fpst = fpstp; \
8838     float##fsz tmp; \
8839     tmp = itype##_to_##float##fsz(x, fpst); \
8840     return float##fsz##_scalbn(tmp, -(int)shift, fpst); \
8841 }
8842
8843 /* Notice that we want only input-denormal exception flags from the
8844  * scalbn operation: the other possible flags (overflow+inexact if
8845  * we overflow to infinity, output-denormal) aren't correct for the
8846  * complete scale-and-convert operation.
8847  */
8848 #define VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, round) \
8849 uint##isz##_t HELPER(vfp_to##name##p##round)(float##fsz x, \
8850                                              uint32_t shift, \
8851                                              void *fpstp) \
8852 { \
8853     float_status *fpst = fpstp; \
8854     int old_exc_flags = get_float_exception_flags(fpst); \
8855     float##fsz tmp; \
8856     if (float##fsz##_is_any_nan(x)) { \
8857         float_raise(float_flag_invalid, fpst); \
8858         return 0; \
8859     } \
8860     tmp = float##fsz##_scalbn(x, shift, fpst); \
8861     old_exc_flags |= get_float_exception_flags(fpst) \
8862         & float_flag_input_denormal; \
8863     set_float_exception_flags(old_exc_flags, fpst); \
8864     return float##fsz##_to_##itype##round(tmp, fpst); \
8865 }
8866
8867 #define VFP_CONV_FIX(name, p, fsz, isz, itype)                   \
8868 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
8869 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, _round_to_zero) \
8870 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
8871
8872 #define VFP_CONV_FIX_A64(name, p, fsz, isz, itype)               \
8873 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
8874 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
8875
8876 VFP_CONV_FIX(sh, d, 64, 64, int16)
8877 VFP_CONV_FIX(sl, d, 64, 64, int32)
8878 VFP_CONV_FIX_A64(sq, d, 64, 64, int64)
8879 VFP_CONV_FIX(uh, d, 64, 64, uint16)
8880 VFP_CONV_FIX(ul, d, 64, 64, uint32)
8881 VFP_CONV_FIX_A64(uq, d, 64, 64, uint64)
8882 VFP_CONV_FIX(sh, s, 32, 32, int16)
8883 VFP_CONV_FIX(sl, s, 32, 32, int32)
8884 VFP_CONV_FIX_A64(sq, s, 32, 64, int64)
8885 VFP_CONV_FIX(uh, s, 32, 32, uint16)
8886 VFP_CONV_FIX(ul, s, 32, 32, uint32)
8887 VFP_CONV_FIX_A64(uq, s, 32, 64, uint64)
8888 #undef VFP_CONV_FIX
8889 #undef VFP_CONV_FIX_FLOAT
8890 #undef VFP_CONV_FLOAT_FIX_ROUND
8891
8892 /* Set the current fp rounding mode and return the old one.
8893  * The argument is a softfloat float_round_ value.
8894  */
8895 uint32_t HELPER(set_rmode)(uint32_t rmode, CPUARMState *env)
8896 {
8897     float_status *fp_status = &env->vfp.fp_status;
8898
8899     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
8900     set_float_rounding_mode(rmode, fp_status);
8901
8902     return prev_rmode;
8903 }
8904
8905 /* Set the current fp rounding mode in the standard fp status and return
8906  * the old one. This is for NEON instructions that need to change the
8907  * rounding mode but wish to use the standard FPSCR values for everything
8908  * else. Always set the rounding mode back to the correct value after
8909  * modifying it.
8910  * The argument is a softfloat float_round_ value.
8911  */
8912 uint32_t HELPER(set_neon_rmode)(uint32_t rmode, CPUARMState *env)
8913 {
8914     float_status *fp_status = &env->vfp.standard_fp_status;
8915
8916     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
8917     set_float_rounding_mode(rmode, fp_status);
8918
8919     return prev_rmode;
8920 }
8921
8922 /* Half precision conversions.  */
8923 static float32 do_fcvt_f16_to_f32(uint32_t a, CPUARMState *env, float_status *s)
8924 {
8925     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8926     float32 r = float16_to_float32(make_float16(a), ieee, s);
8927     if (ieee) {
8928         return float32_maybe_silence_nan(r, s);
8929     }
8930     return r;
8931 }
8932
8933 static uint32_t do_fcvt_f32_to_f16(float32 a, CPUARMState *env, float_status *s)
8934 {
8935     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8936     float16 r = float32_to_float16(a, ieee, s);
8937     if (ieee) {
8938         r = float16_maybe_silence_nan(r, s);
8939     }
8940     return float16_val(r);
8941 }
8942
8943 float32 HELPER(neon_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
8944 {
8945     return do_fcvt_f16_to_f32(a, env, &env->vfp.standard_fp_status);
8946 }
8947
8948 uint32_t HELPER(neon_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
8949 {
8950     return do_fcvt_f32_to_f16(a, env, &env->vfp.standard_fp_status);
8951 }
8952
8953 float32 HELPER(vfp_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
8954 {
8955     return do_fcvt_f16_to_f32(a, env, &env->vfp.fp_status);
8956 }
8957
8958 uint32_t HELPER(vfp_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
8959 {
8960     return do_fcvt_f32_to_f16(a, env, &env->vfp.fp_status);
8961 }
8962
8963 float64 HELPER(vfp_fcvt_f16_to_f64)(uint32_t a, CPUARMState *env)
8964 {
8965     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8966     float64 r = float16_to_float64(make_float16(a), ieee, &env->vfp.fp_status);
8967     if (ieee) {
8968         return float64_maybe_silence_nan(r, &env->vfp.fp_status);
8969     }
8970     return r;
8971 }
8972
8973 uint32_t HELPER(vfp_fcvt_f64_to_f16)(float64 a, CPUARMState *env)
8974 {
8975     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
8976     float16 r = float64_to_float16(a, ieee, &env->vfp.fp_status);
8977     if (ieee) {
8978         r = float16_maybe_silence_nan(r, &env->vfp.fp_status);
8979     }
8980     return float16_val(r);
8981 }
8982
8983 #define float32_two make_float32(0x40000000)
8984 #define float32_three make_float32(0x40400000)
8985 #define float32_one_point_five make_float32(0x3fc00000)
8986
8987 float32 HELPER(recps_f32)(float32 a, float32 b, CPUARMState *env)
8988 {
8989     float_status *s = &env->vfp.standard_fp_status;
8990     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
8991         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
8992         if (!(float32_is_zero(a) || float32_is_zero(b))) {
8993             float_raise(float_flag_input_denormal, s);
8994         }
8995         return float32_two;
8996     }
8997     return float32_sub(float32_two, float32_mul(a, b, s), s);
8998 }
8999
9000 float32 HELPER(rsqrts_f32)(float32 a, float32 b, CPUARMState *env)
9001 {
9002     float_status *s = &env->vfp.standard_fp_status;
9003     float32 product;
9004     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
9005         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
9006         if (!(float32_is_zero(a) || float32_is_zero(b))) {
9007             float_raise(float_flag_input_denormal, s);
9008         }
9009         return float32_one_point_five;
9010     }
9011     product = float32_mul(a, b, s);
9012     return float32_div(float32_sub(float32_three, product, s), float32_two, s);
9013 }
9014
9015 /* NEON helpers.  */
9016
9017 /* Constants 256 and 512 are used in some helpers; we avoid relying on
9018  * int->float conversions at run-time.  */
9019 #define float64_256 make_float64(0x4070000000000000LL)
9020 #define float64_512 make_float64(0x4080000000000000LL)
9021 #define float32_maxnorm make_float32(0x7f7fffff)
9022 #define float64_maxnorm make_float64(0x7fefffffffffffffLL)
9023
9024 /* Reciprocal functions
9025  *
9026  * The algorithm that must be used to calculate the estimate
9027  * is specified by the ARM ARM, see FPRecipEstimate()
9028  */
9029
9030 static float64 recip_estimate(float64 a, float_status *real_fp_status)
9031 {
9032     /* These calculations mustn't set any fp exception flags,
9033      * so we use a local copy of the fp_status.
9034      */
9035     float_status dummy_status = *real_fp_status;
9036     float_status *s = &dummy_status;
9037     /* q = (int)(a * 512.0) */
9038     float64 q = float64_mul(float64_512, a, s);
9039     int64_t q_int = float64_to_int64_round_to_zero(q, s);
9040
9041     /* r = 1.0 / (((double)q + 0.5) / 512.0) */
9042     q = int64_to_float64(q_int, s);
9043     q = float64_add(q, float64_half, s);
9044     q = float64_div(q, float64_512, s);
9045     q = float64_div(float64_one, q, s);
9046
9047     /* s = (int)(256.0 * r + 0.5) */
9048     q = float64_mul(q, float64_256, s);
9049     q = float64_add(q, float64_half, s);
9050     q_int = float64_to_int64_round_to_zero(q, s);
9051
9052     /* return (double)s / 256.0 */
9053     return float64_div(int64_to_float64(q_int, s), float64_256, s);
9054 }
9055
9056 /* Common wrapper to call recip_estimate */
9057 static float64 call_recip_estimate(float64 num, int off, float_status *fpst)
9058 {
9059     uint64_t val64 = float64_val(num);
9060     uint64_t frac = extract64(val64, 0, 52);
9061     int64_t exp = extract64(val64, 52, 11);
9062     uint64_t sbit;
9063     float64 scaled, estimate;
9064
9065     /* Generate the scaled number for the estimate function */
9066     if (exp == 0) {
9067         if (extract64(frac, 51, 1) == 0) {
9068             exp = -1;
9069             frac = extract64(frac, 0, 50) << 2;
9070         } else {
9071             frac = extract64(frac, 0, 51) << 1;
9072         }
9073     }
9074
9075     /* scaled = '0' : '01111111110' : fraction<51:44> : Zeros(44); */
9076     scaled = make_float64((0x3feULL << 52)
9077                           | extract64(frac, 44, 8) << 44);
9078
9079     estimate = recip_estimate(scaled, fpst);
9080
9081     /* Build new result */
9082     val64 = float64_val(estimate);
9083     sbit = 0x8000000000000000ULL & val64;
9084     exp = off - exp;
9085     frac = extract64(val64, 0, 52);
9086
9087     if (exp == 0) {
9088         frac = 1ULL << 51 | extract64(frac, 1, 51);
9089     } else if (exp == -1) {
9090         frac = 1ULL << 50 | extract64(frac, 2, 50);
9091         exp = 0;
9092     }
9093
9094     return make_float64(sbit | (exp << 52) | frac);
9095 }
9096
9097 static bool round_to_inf(float_status *fpst, bool sign_bit)
9098 {
9099     switch (fpst->float_rounding_mode) {
9100     case float_round_nearest_even: /* Round to Nearest */
9101         return true;
9102     case float_round_up: /* Round to +Inf */
9103         return !sign_bit;
9104     case float_round_down: /* Round to -Inf */
9105         return sign_bit;
9106     case float_round_to_zero: /* Round to Zero */
9107         return false;
9108     }
9109
9110     g_assert_not_reached();
9111 }
9112
9113 float32 HELPER(recpe_f32)(float32 input, void *fpstp)
9114 {
9115     float_status *fpst = fpstp;
9116     float32 f32 = float32_squash_input_denormal(input, fpst);
9117     uint32_t f32_val = float32_val(f32);
9118     uint32_t f32_sbit = 0x80000000ULL & f32_val;
9119     int32_t f32_exp = extract32(f32_val, 23, 8);
9120     uint32_t f32_frac = extract32(f32_val, 0, 23);
9121     float64 f64, r64;
9122     uint64_t r64_val;
9123     int64_t r64_exp;
9124     uint64_t r64_frac;
9125
9126     if (float32_is_any_nan(f32)) {
9127         float32 nan = f32;
9128         if (float32_is_signaling_nan(f32, fpst)) {
9129             float_raise(float_flag_invalid, fpst);
9130             nan = float32_maybe_silence_nan(f32, fpst);
9131         }
9132         if (fpst->default_nan_mode) {
9133             nan =  float32_default_nan(fpst);
9134         }
9135         return nan;
9136     } else if (float32_is_infinity(f32)) {
9137         return float32_set_sign(float32_zero, float32_is_neg(f32));
9138     } else if (float32_is_zero(f32)) {
9139         float_raise(float_flag_divbyzero, fpst);
9140         return float32_set_sign(float32_infinity, float32_is_neg(f32));
9141     } else if ((f32_val & ~(1ULL << 31)) < (1ULL << 21)) {
9142         /* Abs(value) < 2.0^-128 */
9143         float_raise(float_flag_overflow | float_flag_inexact, fpst);
9144         if (round_to_inf(fpst, f32_sbit)) {
9145             return float32_set_sign(float32_infinity, float32_is_neg(f32));
9146         } else {
9147             return float32_set_sign(float32_maxnorm, float32_is_neg(f32));
9148         }
9149     } else if (f32_exp >= 253 && fpst->flush_to_zero) {
9150         float_raise(float_flag_underflow, fpst);
9151         return float32_set_sign(float32_zero, float32_is_neg(f32));
9152     }
9153
9154
9155     f64 = make_float64(((int64_t)(f32_exp) << 52) | (int64_t)(f32_frac) << 29);
9156     r64 = call_recip_estimate(f64, 253, fpst);
9157     r64_val = float64_val(r64);
9158     r64_exp = extract64(r64_val, 52, 11);
9159     r64_frac = extract64(r64_val, 0, 52);
9160
9161     /* result = sign : result_exp<7:0> : fraction<51:29>; */
9162     return make_float32(f32_sbit |
9163                         (r64_exp & 0xff) << 23 |
9164                         extract64(r64_frac, 29, 24));
9165 }
9166
9167 float64 HELPER(recpe_f64)(float64 input, void *fpstp)
9168 {
9169     float_status *fpst = fpstp;
9170     float64 f64 = float64_squash_input_denormal(input, fpst);
9171     uint64_t f64_val = float64_val(f64);
9172     uint64_t f64_sbit = 0x8000000000000000ULL & f64_val;
9173     int64_t f64_exp = extract64(f64_val, 52, 11);
9174     float64 r64;
9175     uint64_t r64_val;
9176     int64_t r64_exp;
9177     uint64_t r64_frac;
9178
9179     /* Deal with any special cases */
9180     if (float64_is_any_nan(f64)) {
9181         float64 nan = f64;
9182         if (float64_is_signaling_nan(f64, fpst)) {
9183             float_raise(float_flag_invalid, fpst);
9184             nan = float64_maybe_silence_nan(f64, fpst);
9185         }
9186         if (fpst->default_nan_mode) {
9187             nan =  float64_default_nan(fpst);
9188         }
9189         return nan;
9190     } else if (float64_is_infinity(f64)) {
9191         return float64_set_sign(float64_zero, float64_is_neg(f64));
9192     } else if (float64_is_zero(f64)) {
9193         float_raise(float_flag_divbyzero, fpst);
9194         return float64_set_sign(float64_infinity, float64_is_neg(f64));
9195     } else if ((f64_val & ~(1ULL << 63)) < (1ULL << 50)) {
9196         /* Abs(value) < 2.0^-1024 */
9197         float_raise(float_flag_overflow | float_flag_inexact, fpst);
9198         if (round_to_inf(fpst, f64_sbit)) {
9199             return float64_set_sign(float64_infinity, float64_is_neg(f64));
9200         } else {
9201             return float64_set_sign(float64_maxnorm, float64_is_neg(f64));
9202         }
9203     } else if (f64_exp >= 2045 && fpst->flush_to_zero) {
9204         float_raise(float_flag_underflow, fpst);
9205         return float64_set_sign(float64_zero, float64_is_neg(f64));
9206     }
9207
9208     r64 = call_recip_estimate(f64, 2045, fpst);
9209     r64_val = float64_val(r64);
9210     r64_exp = extract64(r64_val, 52, 11);
9211     r64_frac = extract64(r64_val, 0, 52);
9212
9213     /* result = sign : result_exp<10:0> : fraction<51:0> */
9214     return make_float64(f64_sbit |
9215                         ((r64_exp & 0x7ff) << 52) |
9216                         r64_frac);
9217 }
9218
9219 /* The algorithm that must be used to calculate the estimate
9220  * is specified by the ARM ARM.
9221  */
9222 static float64 recip_sqrt_estimate(float64 a, float_status *real_fp_status)
9223 {
9224     /* These calculations mustn't set any fp exception flags,
9225      * so we use a local copy of the fp_status.
9226      */
9227     float_status dummy_status = *real_fp_status;
9228     float_status *s = &dummy_status;
9229     float64 q;
9230     int64_t q_int;
9231
9232     if (float64_lt(a, float64_half, s)) {
9233         /* range 0.25 <= a < 0.5 */
9234
9235         /* a in units of 1/512 rounded down */
9236         /* q0 = (int)(a * 512.0);  */
9237         q = float64_mul(float64_512, a, s);
9238         q_int = float64_to_int64_round_to_zero(q, s);
9239
9240         /* reciprocal root r */
9241         /* r = 1.0 / sqrt(((double)q0 + 0.5) / 512.0);  */
9242         q = int64_to_float64(q_int, s);
9243         q = float64_add(q, float64_half, s);
9244         q = float64_div(q, float64_512, s);
9245         q = float64_sqrt(q, s);
9246         q = float64_div(float64_one, q, s);
9247     } else {
9248         /* range 0.5 <= a < 1.0 */
9249
9250         /* a in units of 1/256 rounded down */
9251         /* q1 = (int)(a * 256.0); */
9252         q = float64_mul(float64_256, a, s);
9253         int64_t q_int = float64_to_int64_round_to_zero(q, s);
9254
9255         /* reciprocal root r */
9256         /* r = 1.0 /sqrt(((double)q1 + 0.5) / 256); */
9257         q = int64_to_float64(q_int, s);
9258         q = float64_add(q, float64_half, s);
9259         q = float64_div(q, float64_256, s);
9260         q = float64_sqrt(q, s);
9261         q = float64_div(float64_one, q, s);
9262     }
9263     /* r in units of 1/256 rounded to nearest */
9264     /* s = (int)(256.0 * r + 0.5); */
9265
9266     q = float64_mul(q, float64_256,s );
9267     q = float64_add(q, float64_half, s);
9268     q_int = float64_to_int64_round_to_zero(q, s);
9269
9270     /* return (double)s / 256.0;*/
9271     return float64_div(int64_to_float64(q_int, s), float64_256, s);
9272 }
9273
9274 float32 HELPER(rsqrte_f32)(float32 input, void *fpstp)
9275 {
9276     float_status *s = fpstp;
9277     float32 f32 = float32_squash_input_denormal(input, s);
9278     uint32_t val = float32_val(f32);
9279     uint32_t f32_sbit = 0x80000000 & val;
9280     int32_t f32_exp = extract32(val, 23, 8);
9281     uint32_t f32_frac = extract32(val, 0, 23);
9282     uint64_t f64_frac;
9283     uint64_t val64;
9284     int result_exp;
9285     float64 f64;
9286
9287     if (float32_is_any_nan(f32)) {
9288         float32 nan = f32;
9289         if (float32_is_signaling_nan(f32, s)) {
9290             float_raise(float_flag_invalid, s);
9291             nan = float32_maybe_silence_nan(f32, s);
9292         }
9293         if (s->default_nan_mode) {
9294             nan =  float32_default_nan(s);
9295         }
9296         return nan;
9297     } else if (float32_is_zero(f32)) {
9298         float_raise(float_flag_divbyzero, s);
9299         return float32_set_sign(float32_infinity, float32_is_neg(f32));
9300     } else if (float32_is_neg(f32)) {
9301         float_raise(float_flag_invalid, s);
9302         return float32_default_nan(s);
9303     } else if (float32_is_infinity(f32)) {
9304         return float32_zero;
9305     }
9306
9307     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
9308      * preserving the parity of the exponent.  */
9309
9310     f64_frac = ((uint64_t) f32_frac) << 29;
9311     if (f32_exp == 0) {
9312         while (extract64(f64_frac, 51, 1) == 0) {
9313             f64_frac = f64_frac << 1;
9314             f32_exp = f32_exp-1;
9315         }
9316         f64_frac = extract64(f64_frac, 0, 51) << 1;
9317     }
9318
9319     if (extract64(f32_exp, 0, 1) == 0) {
9320         f64 = make_float64(((uint64_t) f32_sbit) << 32
9321                            | (0x3feULL << 52)
9322                            | f64_frac);
9323     } else {
9324         f64 = make_float64(((uint64_t) f32_sbit) << 32
9325                            | (0x3fdULL << 52)
9326                            | f64_frac);
9327     }
9328
9329     result_exp = (380 - f32_exp) / 2;
9330
9331     f64 = recip_sqrt_estimate(f64, s);
9332
9333     val64 = float64_val(f64);
9334
9335     val = ((result_exp & 0xff) << 23)
9336         | ((val64 >> 29)  & 0x7fffff);
9337     return make_float32(val);
9338 }
9339
9340 float64 HELPER(rsqrte_f64)(float64 input, void *fpstp)
9341 {
9342     float_status *s = fpstp;
9343     float64 f64 = float64_squash_input_denormal(input, s);
9344     uint64_t val = float64_val(f64);
9345     uint64_t f64_sbit = 0x8000000000000000ULL & val;
9346     int64_t f64_exp = extract64(val, 52, 11);
9347     uint64_t f64_frac = extract64(val, 0, 52);
9348     int64_t result_exp;
9349     uint64_t result_frac;
9350
9351     if (float64_is_any_nan(f64)) {
9352         float64 nan = f64;
9353         if (float64_is_signaling_nan(f64, s)) {
9354             float_raise(float_flag_invalid, s);
9355             nan = float64_maybe_silence_nan(f64, s);
9356         }
9357         if (s->default_nan_mode) {
9358             nan =  float64_default_nan(s);
9359         }
9360         return nan;
9361     } else if (float64_is_zero(f64)) {
9362         float_raise(float_flag_divbyzero, s);
9363         return float64_set_sign(float64_infinity, float64_is_neg(f64));
9364     } else if (float64_is_neg(f64)) {
9365         float_raise(float_flag_invalid, s);
9366         return float64_default_nan(s);
9367     } else if (float64_is_infinity(f64)) {
9368         return float64_zero;
9369     }
9370
9371     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
9372      * preserving the parity of the exponent.  */
9373
9374     if (f64_exp == 0) {
9375         while (extract64(f64_frac, 51, 1) == 0) {
9376             f64_frac = f64_frac << 1;
9377             f64_exp = f64_exp - 1;
9378         }
9379         f64_frac = extract64(f64_frac, 0, 51) << 1;
9380     }
9381
9382     if (extract64(f64_exp, 0, 1) == 0) {
9383         f64 = make_float64(f64_sbit
9384                            | (0x3feULL << 52)
9385                            | f64_frac);
9386     } else {
9387         f64 = make_float64(f64_sbit
9388                            | (0x3fdULL << 52)
9389                            | f64_frac);
9390     }
9391
9392     result_exp = (3068 - f64_exp) / 2;
9393
9394     f64 = recip_sqrt_estimate(f64, s);
9395
9396     result_frac = extract64(float64_val(f64), 0, 52);
9397
9398     return make_float64(f64_sbit |
9399                         ((result_exp & 0x7ff) << 52) |
9400                         result_frac);
9401 }
9402
9403 uint32_t HELPER(recpe_u32)(uint32_t a, void *fpstp)
9404 {
9405     float_status *s = fpstp;
9406     float64 f64;
9407
9408     if ((a & 0x80000000) == 0) {
9409         return 0xffffffff;
9410     }
9411
9412     f64 = make_float64((0x3feULL << 52)
9413                        | ((int64_t)(a & 0x7fffffff) << 21));
9414
9415     f64 = recip_estimate(f64, s);
9416
9417     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
9418 }
9419
9420 uint32_t HELPER(rsqrte_u32)(uint32_t a, void *fpstp)
9421 {
9422     float_status *fpst = fpstp;
9423     float64 f64;
9424
9425     if ((a & 0xc0000000) == 0) {
9426         return 0xffffffff;
9427     }
9428
9429     if (a & 0x80000000) {
9430         f64 = make_float64((0x3feULL << 52)
9431                            | ((uint64_t)(a & 0x7fffffff) << 21));
9432     } else { /* bits 31-30 == '01' */
9433         f64 = make_float64((0x3fdULL << 52)
9434                            | ((uint64_t)(a & 0x3fffffff) << 22));
9435     }
9436
9437     f64 = recip_sqrt_estimate(f64, fpst);
9438
9439     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
9440 }
9441
9442 /* VFPv4 fused multiply-accumulate */
9443 float32 VFP_HELPER(muladd, s)(float32 a, float32 b, float32 c, void *fpstp)
9444 {
9445     float_status *fpst = fpstp;
9446     return float32_muladd(a, b, c, 0, fpst);
9447 }
9448
9449 float64 VFP_HELPER(muladd, d)(float64 a, float64 b, float64 c, void *fpstp)
9450 {
9451     float_status *fpst = fpstp;
9452     return float64_muladd(a, b, c, 0, fpst);
9453 }
9454
9455 /* ARMv8 round to integral */
9456 float32 HELPER(rints_exact)(float32 x, void *fp_status)
9457 {
9458     return float32_round_to_int(x, fp_status);
9459 }
9460
9461 float64 HELPER(rintd_exact)(float64 x, void *fp_status)
9462 {
9463     return float64_round_to_int(x, fp_status);
9464 }
9465
9466 float32 HELPER(rints)(float32 x, void *fp_status)
9467 {
9468     int old_flags = get_float_exception_flags(fp_status), new_flags;
9469     float32 ret;
9470
9471     ret = float32_round_to_int(x, fp_status);
9472
9473     /* Suppress any inexact exceptions the conversion produced */
9474     if (!(old_flags & float_flag_inexact)) {
9475         new_flags = get_float_exception_flags(fp_status);
9476         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
9477     }
9478
9479     return ret;
9480 }
9481
9482 float64 HELPER(rintd)(float64 x, void *fp_status)
9483 {
9484     int old_flags = get_float_exception_flags(fp_status), new_flags;
9485     float64 ret;
9486
9487     ret = float64_round_to_int(x, fp_status);
9488
9489     new_flags = get_float_exception_flags(fp_status);
9490
9491     /* Suppress any inexact exceptions the conversion produced */
9492     if (!(old_flags & float_flag_inexact)) {
9493         new_flags = get_float_exception_flags(fp_status);
9494         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
9495     }
9496
9497     return ret;
9498 }
9499
9500 /* Convert ARM rounding mode to softfloat */
9501 int arm_rmode_to_sf(int rmode)
9502 {
9503     switch (rmode) {
9504     case FPROUNDING_TIEAWAY:
9505         rmode = float_round_ties_away;
9506         break;
9507     case FPROUNDING_ODD:
9508         /* FIXME: add support for TIEAWAY and ODD */
9509         qemu_log_mask(LOG_UNIMP, "arm: unimplemented rounding mode: %d\n",
9510                       rmode);
9511     case FPROUNDING_TIEEVEN:
9512     default:
9513         rmode = float_round_nearest_even;
9514         break;
9515     case FPROUNDING_POSINF:
9516         rmode = float_round_up;
9517         break;
9518     case FPROUNDING_NEGINF:
9519         rmode = float_round_down;
9520         break;
9521     case FPROUNDING_ZERO:
9522         rmode = float_round_to_zero;
9523         break;
9524     }
9525     return rmode;
9526 }
9527
9528 /* CRC helpers.
9529  * The upper bytes of val (above the number specified by 'bytes') must have
9530  * been zeroed out by the caller.
9531  */
9532 uint32_t HELPER(crc32)(uint32_t acc, uint32_t val, uint32_t bytes)
9533 {
9534     uint8_t buf[4];
9535
9536     stl_le_p(buf, val);
9537
9538     /* zlib crc32 converts the accumulator and output to one's complement.  */
9539     return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
9540 }
9541
9542 uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
9543 {
9544     uint8_t buf[4];
9545
9546     stl_le_p(buf, val);
9547
9548     /* Linux crc32c converts the output to one's complement.  */
9549     return crc32c(acc, buf, bytes) ^ 0xffffffff;
9550 }