target-arm/helper.c

   1 #include "cpu.h"
   2 #include "internals.h"
   3 #include "exec/gdbstub.h"
   4 #include "exec/helper-proto.h"
   5 #include "qemu/host-utils.h"
   6 #include "sysemu/arch_init.h"
   7 #include "sysemu/sysemu.h"
   8 #include "qemu/bitops.h"
   9 #include "qemu/crc32c.h"
  10 #include "exec/cpu_ldst.h"
  11 #include "arm_ldst.h"
  12 #include <zlib.h> /* For crc32 */
  13
  14 #ifndef CONFIG_USER_ONLY
  15 static inline int get_phys_addr(CPUARMState *env, target_ulong address,
  16                                 int access_type, int is_user,
  17                                 hwaddr *phys_ptr, int *prot,
  18                                 target_ulong *page_size);
  19
  20 /* Definitions for the PMCCNTR and PMCR registers */
  21 #define PMCRD   0x8
  22 #define PMCRC   0x4
  23 #define PMCRE   0x1
  24 #endif
  25
  26 static int vfp_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  27 {
  28     int nregs;
  29
  30     /* VFP data registers are always little-endian.  */
  31     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  32     if (reg < nregs) {
  33         stfq_le_p(buf, env->vfp.regs[reg]);
  34         return 8;
  35     }
  36     if (arm_feature(env, ARM_FEATURE_NEON)) {
  37         /* Aliases for Q regs.  */
  38         nregs += 16;
  39         if (reg < nregs) {
  40             stfq_le_p(buf, env->vfp.regs[(reg - 32) * 2]);
  41             stfq_le_p(buf + 8, env->vfp.regs[(reg - 32) * 2 + 1]);
  42             return 16;
  43         }
  44     }
  45     switch (reg - nregs) {
  46     case 0: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSID]); return 4;
  47     case 1: stl_p(buf, env->vfp.xregs[ARM_VFP_FPSCR]); return 4;
  48     case 2: stl_p(buf, env->vfp.xregs[ARM_VFP_FPEXC]); return 4;
  49     }
  50     return 0;
  51 }
  52
  53 static int vfp_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
  54 {
  55     int nregs;
  56
  57     nregs = arm_feature(env, ARM_FEATURE_VFP3) ? 32 : 16;
  58     if (reg < nregs) {
  59         env->vfp.regs[reg] = ldfq_le_p(buf);
  60         return 8;
  61     }
  62     if (arm_feature(env, ARM_FEATURE_NEON)) {
  63         nregs += 16;
  64         if (reg < nregs) {
  65             env->vfp.regs[(reg - 32) * 2] = ldfq_le_p(buf);
  66             env->vfp.regs[(reg - 32) * 2 + 1] = ldfq_le_p(buf + 8);
  67             return 16;
  68         }
  69     }
  70     switch (reg - nregs) {
  71     case 0: env->vfp.xregs[ARM_VFP_FPSID] = ldl_p(buf); return 4;
  72     case 1: env->vfp.xregs[ARM_VFP_FPSCR] = ldl_p(buf); return 4;
  73     case 2: env->vfp.xregs[ARM_VFP_FPEXC] = ldl_p(buf) & (1 << 30); return 4;
  74     }
  75     return 0;
  76 }
  77
  78 static int aarch64_fpu_gdb_get_reg(CPUARMState *env, uint8_t *buf, int reg)
  79 {
  80     switch (reg) {
  81     case 0 ... 31:
  82         /* 128 bit FP register */
  83         stfq_le_p(buf, env->vfp.regs[reg * 2]);
  84         stfq_le_p(buf + 8, env->vfp.regs[reg * 2 + 1]);
  85         return 16;
  86     case 32:
  87         /* FPSR */
  88         stl_p(buf, vfp_get_fpsr(env));
  89         return 4;
  90     case 33:
  91         /* FPCR */
  92         stl_p(buf, vfp_get_fpcr(env));
  93         return 4;
  94     default:
  95         return 0;
  96     }
  97 }
  98
  99 static int aarch64_fpu_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg)
 100 {
 101     switch (reg) {
 102     case 0 ... 31:
 103         /* 128 bit FP register */
 104         env->vfp.regs[reg * 2] = ldfq_le_p(buf);
 105         env->vfp.regs[reg * 2 + 1] = ldfq_le_p(buf + 8);
 106         return 16;
 107     case 32:
 108         /* FPSR */
 109         vfp_set_fpsr(env, ldl_p(buf));
 110         return 4;
 111     case 33:
 112         /* FPCR */
 113         vfp_set_fpcr(env, ldl_p(buf));
 114         return 4;
 115     default:
 116         return 0;
 117     }
 118 }
 119
 120 static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri)
 121 {
 122     if (cpreg_field_is_64bit(ri)) {
 123         return CPREG_FIELD64(env, ri);
 124     } else {
 125         return CPREG_FIELD32(env, ri);
 126     }
 127 }
 128
 129 static void raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
 130                       uint64_t value)
 131 {
 132     if (cpreg_field_is_64bit(ri)) {
 133         CPREG_FIELD64(env, ri) = value;
 134     } else {
 135         CPREG_FIELD32(env, ri) = value;
 136     }
 137 }
 138
 139 static void *raw_ptr(CPUARMState *env, const ARMCPRegInfo *ri)
 140 {
 141     return (char *)env + ri->fieldoffset;
 142 }
 143
 144 static uint64_t read_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri)
 145 {
 146     /* Raw read of a coprocessor register (as needed for migration, etc). */
 147     if (ri->type & ARM_CP_CONST) {
 148         return ri->resetvalue;
 149     } else if (ri->raw_readfn) {
 150         return ri->raw_readfn(env, ri);
 151     } else if (ri->readfn) {
 152         return ri->readfn(env, ri);
 153     } else {
 154         return raw_read(env, ri);
 155     }
 156 }
 157
 158 static void write_raw_cp_reg(CPUARMState *env, const ARMCPRegInfo *ri,
 159                              uint64_t v)
 160 {
 161     /* Raw write of a coprocessor register (as needed for migration, etc).
 162      * Note that constant registers are treated as write-ignored; the
 163      * caller should check for success by whether a readback gives the
 164      * value written.
 165      */
 166     if (ri->type & ARM_CP_CONST) {
 167         return;
 168     } else if (ri->raw_writefn) {
 169         ri->raw_writefn(env, ri, v);
 170     } else if (ri->writefn) {
 171         ri->writefn(env, ri, v);
 172     } else {
 173         raw_write(env, ri, v);
 174     }
 175 }
 176
 177 bool write_cpustate_to_list(ARMCPU *cpu)
 178 {
 179     /* Write the coprocessor state from cpu->env to the (index,value) list. */
 180     int i;
 181     bool ok = true;
 182
 183     for (i = 0; i < cpu->cpreg_array_len; i++) {
 184         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 185         const ARMCPRegInfo *ri;
 186
 187         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 188         if (!ri) {
 189             ok = false;
 190             continue;
 191         }
 192         if (ri->type & ARM_CP_NO_MIGRATE) {
 193             continue;
 194         }
 195         cpu->cpreg_values[i] = read_raw_cp_reg(&cpu->env, ri);
 196     }
 197     return ok;
 198 }
 199
 200 bool write_list_to_cpustate(ARMCPU *cpu)
 201 {
 202     int i;
 203     bool ok = true;
 204
 205     for (i = 0; i < cpu->cpreg_array_len; i++) {
 206         uint32_t regidx = kvm_to_cpreg_id(cpu->cpreg_indexes[i]);
 207         uint64_t v = cpu->cpreg_values[i];
 208         const ARMCPRegInfo *ri;
 209
 210         ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 211         if (!ri) {
 212             ok = false;
 213             continue;
 214         }
 215         if (ri->type & ARM_CP_NO_MIGRATE) {
 216             continue;
 217         }
 218         /* Write value and confirm it reads back as written
 219          * (to catch read-only registers and partially read-only
 220          * registers where the incoming migration value doesn't match)
 221          */
 222         write_raw_cp_reg(&cpu->env, ri, v);
 223         if (read_raw_cp_reg(&cpu->env, ri) != v) {
 224             ok = false;
 225         }
 226     }
 227     return ok;
 228 }
 229
 230 static void add_cpreg_to_list(gpointer key, gpointer opaque)
 231 {
 232     ARMCPU *cpu = opaque;
 233     uint64_t regidx;
 234     const ARMCPRegInfo *ri;
 235
 236     regidx = *(uint32_t *)key;
 237     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 238
 239     if (!(ri->type & ARM_CP_NO_MIGRATE)) {
 240         cpu->cpreg_indexes[cpu->cpreg_array_len] = cpreg_to_kvm_id(regidx);
 241         /* The value array need not be initialized at this point */
 242         cpu->cpreg_array_len++;
 243     }
 244 }
 245
 246 static void count_cpreg(gpointer key, gpointer opaque)
 247 {
 248     ARMCPU *cpu = opaque;
 249     uint64_t regidx;
 250     const ARMCPRegInfo *ri;
 251
 252     regidx = *(uint32_t *)key;
 253     ri = get_arm_cp_reginfo(cpu->cp_regs, regidx);
 254
 255     if (!(ri->type & ARM_CP_NO_MIGRATE)) {
 256         cpu->cpreg_array_len++;
 257     }
 258 }
 259
 260 static gint cpreg_key_compare(gconstpointer a, gconstpointer b)
 261 {
 262     uint64_t aidx = cpreg_to_kvm_id(*(uint32_t *)a);
 263     uint64_t bidx = cpreg_to_kvm_id(*(uint32_t *)b);
 264
 265     if (aidx > bidx) {
 266         return 1;
 267     }
 268     if (aidx < bidx) {
 269         return -1;
 270     }
 271     return 0;
 272 }
 273
 274 static void cpreg_make_keylist(gpointer key, gpointer value, gpointer udata)
 275 {
 276     GList **plist = udata;
 277
 278     *plist = g_list_prepend(*plist, key);
 279 }
 280
 281 void init_cpreg_list(ARMCPU *cpu)
 282 {
 283     /* Initialise the cpreg_tuples[] array based on the cp_regs hash.
 284      * Note that we require cpreg_tuples[] to be sorted by key ID.
 285      */
 286     GList *keys = NULL;
 287     int arraylen;
 288
 289     g_hash_table_foreach(cpu->cp_regs, cpreg_make_keylist, &keys);
 290
 291     keys = g_list_sort(keys, cpreg_key_compare);
 292
 293     cpu->cpreg_array_len = 0;
 294
 295     g_list_foreach(keys, count_cpreg, cpu);
 296
 297     arraylen = cpu->cpreg_array_len;
 298     cpu->cpreg_indexes = g_new(uint64_t, arraylen);
 299     cpu->cpreg_values = g_new(uint64_t, arraylen);
 300     cpu->cpreg_vmstate_indexes = g_new(uint64_t, arraylen);
 301     cpu->cpreg_vmstate_values = g_new(uint64_t, arraylen);
 302     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
 303     cpu->cpreg_array_len = 0;
 304
 305     g_list_foreach(keys, add_cpreg_to_list, cpu);
 306
 307     assert(cpu->cpreg_array_len == arraylen);
 308
 309     g_list_free(keys);
 310 }
 311
 312 static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 313 {
 314     ARMCPU *cpu = arm_env_get_cpu(env);
 315
 316     raw_write(env, ri, value);
 317     tlb_flush(CPU(cpu), 1); /* Flush TLB as domain not tracked in TLB */
 318 }
 319
 320 static void fcse_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 321 {
 322     ARMCPU *cpu = arm_env_get_cpu(env);
 323
 324     if (raw_read(env, ri) != value) {
 325         /* Unlike real hardware the qemu TLB uses virtual addresses,
 326          * not modified virtual addresses, so this causes a TLB flush.
 327          */
 328         tlb_flush(CPU(cpu), 1);
 329         raw_write(env, ri, value);
 330     }
 331 }
 332
 333 static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 334                              uint64_t value)
 335 {
 336     ARMCPU *cpu = arm_env_get_cpu(env);
 337
 338     if (raw_read(env, ri) != value && !arm_feature(env, ARM_FEATURE_MPU)
 339         && !extended_addresses_enabled(env)) {
 340         /* For VMSA (when not using the LPAE long descriptor page table
 341          * format) this register includes the ASID, so do a TLB flush.
 342          * For PMSA it is purely a process ID and no action is needed.
 343          */
 344         tlb_flush(CPU(cpu), 1);
 345     }
 346     raw_write(env, ri, value);
 347 }
 348
 349 static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
 350                           uint64_t value)
 351 {
 352     /* Invalidate all (TLBIALL) */
 353     ARMCPU *cpu = arm_env_get_cpu(env);
 354
 355     tlb_flush(CPU(cpu), 1);
 356 }
 357
 358 static void tlbimva_write(CPUARMState *env, const ARMCPRegInfo *ri,
 359                           uint64_t value)
 360 {
 361     /* Invalidate single TLB entry by MVA and ASID (TLBIMVA) */
 362     ARMCPU *cpu = arm_env_get_cpu(env);
 363
 364     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 365 }
 366
 367 static void tlbiasid_write(CPUARMState *env, const ARMCPRegInfo *ri,
 368                            uint64_t value)
 369 {
 370     /* Invalidate by ASID (TLBIASID) */
 371     ARMCPU *cpu = arm_env_get_cpu(env);
 372
 373     tlb_flush(CPU(cpu), value == 0);
 374 }
 375
 376 static void tlbimvaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
 377                            uint64_t value)
 378 {
 379     /* Invalidate single entry by MVA, all ASIDs (TLBIMVAA) */
 380     ARMCPU *cpu = arm_env_get_cpu(env);
 381
 382     tlb_flush_page(CPU(cpu), value & TARGET_PAGE_MASK);
 383 }
 384
 385 /* IS variants of TLB operations must affect all cores */
 386 static void tlbiall_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 387                              uint64_t value)
 388 {
 389     CPUState *other_cs;
 390
 391     CPU_FOREACH(other_cs) {
 392         tlb_flush(other_cs, 1);
 393     }
 394 }
 395
 396 static void tlbiasid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 397                              uint64_t value)
 398 {
 399     CPUState *other_cs;
 400
 401     CPU_FOREACH(other_cs) {
 402         tlb_flush(other_cs, value == 0);
 403     }
 404 }
 405
 406 static void tlbimva_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 407                              uint64_t value)
 408 {
 409     CPUState *other_cs;
 410
 411     CPU_FOREACH(other_cs) {
 412         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 413     }
 414 }
 415
 416 static void tlbimvaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
 417                              uint64_t value)
 418 {
 419     CPUState *other_cs;
 420
 421     CPU_FOREACH(other_cs) {
 422         tlb_flush_page(other_cs, value & TARGET_PAGE_MASK);
 423     }
 424 }
 425
 426 static const ARMCPRegInfo cp_reginfo[] = {
 427     /* Define the secure and non-secure FCSE identifier CP registers
 428      * separately because there is no secure bank in V8 (no _EL3).  This allows
 429      * the secure register to be properly reset and migrated. There is also no
 430      * v8 EL1 version of the register so the non-secure instance stands alone.
 431      */
 432     { .name = "FCSEIDR(NS)",
 433       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 434       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 435       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_ns),
 436       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 437     { .name = "FCSEIDR(S)",
 438       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 0,
 439       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 440       .fieldoffset = offsetof(CPUARMState, cp15.fcseidr_s),
 441       .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
 442     /* Define the secure and non-secure context identifier CP registers
 443      * separately because there is no secure bank in V8 (no _EL3).  This allows
 444      * the secure register to be properly reset and migrated.  In the
 445      * non-secure case, the 32-bit register will have reset and migration
 446      * disabled during registration as it is handled by the 64-bit instance.
 447      */
 448     { .name = "CONTEXTIDR_EL1", .state = ARM_CP_STATE_BOTH,
 449       .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 450       .access = PL1_RW, .secure = ARM_CP_SECSTATE_NS,
 451       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el[1]),
 452       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 453     { .name = "CONTEXTIDR(S)", .state = ARM_CP_STATE_AA32,
 454       .cp = 15, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
 455       .access = PL1_RW, .secure = ARM_CP_SECSTATE_S,
 456       .fieldoffset = offsetof(CPUARMState, cp15.contextidr_s),
 457       .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
 458     REGINFO_SENTINEL
 459 };
 460
 461 static const ARMCPRegInfo not_v8_cp_reginfo[] = {
 462     /* NB: Some of these registers exist in v8 but with more precise
 463      * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
 464      */
 465     /* MMU Domain access control / MPU write buffer control */
 466     { .name = "DACR",
 467       .cp = 15, .opc1 = CP_ANY, .crn = 3, .crm = CP_ANY, .opc2 = CP_ANY,
 468       .access = PL1_RW, .resetvalue = 0,
 469       .writefn = dacr_write, .raw_writefn = raw_write,
 470       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
 471                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
 472     /* ??? This covers not just the impdef TLB lockdown registers but also
 473      * some v7VMSA registers relating to TEX remap, so it is overly broad.
 474      */
 475     { .name = "TLB_LOCKDOWN", .cp = 15, .crn = 10, .crm = CP_ANY,
 476       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_NOP },
 477     /* Cache maintenance ops; some of this space may be overridden later. */
 478     { .name = "CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
 479       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
 480       .type = ARM_CP_NOP | ARM_CP_OVERRIDE },
 481     REGINFO_SENTINEL
 482 };
 483
 484 static const ARMCPRegInfo not_v6_cp_reginfo[] = {
 485     /* Not all pre-v6 cores implemented this WFI, so this is slightly
 486      * over-broad.
 487      */
 488     { .name = "WFI_v5", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = 2,
 489       .access = PL1_W, .type = ARM_CP_WFI },
 490     REGINFO_SENTINEL
 491 };
 492
 493 static const ARMCPRegInfo not_v7_cp_reginfo[] = {
 494     /* Standard v6 WFI (also used in some pre-v6 cores); not in v7 (which
 495      * is UNPREDICTABLE; we choose to NOP as most implementations do).
 496      */
 497     { .name = "WFI_v6", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 498       .access = PL1_W, .type = ARM_CP_WFI },
 499     /* L1 cache lockdown. Not architectural in v6 and earlier but in practice
 500      * implemented in 926, 946, 1026, 1136, 1176 and 11MPCore. StrongARM and
 501      * OMAPCP will override this space.
 502      */
 503     { .name = "DLOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 0,
 504       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_data),
 505       .resetvalue = 0 },
 506     { .name = "ILOCKDOWN", .cp = 15, .crn = 9, .crm = 0, .opc1 = 0, .opc2 = 1,
 507       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_insn),
 508       .resetvalue = 0 },
 509     /* v6 doesn't have the cache ID registers but Linux reads them anyway */
 510     { .name = "DUMMY", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = CP_ANY,
 511       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
 512       .resetvalue = 0 },
 513     /* We don't implement pre-v7 debug but most CPUs had at least a DBGDIDR;
 514      * implementing it as RAZ means the "debug architecture version" bits
 515      * will read as a reserved value, which should cause Linux to not try
 516      * to use the debug hardware.
 517      */
 518     { .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
 519       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
 520     /* MMU TLB control. Note that the wildcarding means we cover not just
 521      * the unified TLB ops but also the dside/iside/inner-shareable variants.
 522      */
 523     { .name = "TLBIALL", .cp = 15, .crn = 8, .crm = CP_ANY,
 524       .opc1 = CP_ANY, .opc2 = 0, .access = PL1_W, .writefn = tlbiall_write,
 525       .type = ARM_CP_NO_MIGRATE },
 526     { .name = "TLBIMVA", .cp = 15, .crn = 8, .crm = CP_ANY,
 527       .opc1 = CP_ANY, .opc2 = 1, .access = PL1_W, .writefn = tlbimva_write,
 528       .type = ARM_CP_NO_MIGRATE },
 529     { .name = "TLBIASID", .cp = 15, .crn = 8, .crm = CP_ANY,
 530       .opc1 = CP_ANY, .opc2 = 2, .access = PL1_W, .writefn = tlbiasid_write,
 531       .type = ARM_CP_NO_MIGRATE },
 532     { .name = "TLBIMVAA", .cp = 15, .crn = 8, .crm = CP_ANY,
 533       .opc1 = CP_ANY, .opc2 = 3, .access = PL1_W, .writefn = tlbimvaa_write,
 534       .type = ARM_CP_NO_MIGRATE },
 535     REGINFO_SENTINEL
 536 };
 537
 538 static void cpacr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 539                         uint64_t value)
 540 {
 541     uint32_t mask = 0;
 542
 543     /* In ARMv8 most bits of CPACR_EL1 are RES0. */
 544     if (!arm_feature(env, ARM_FEATURE_V8)) {
 545         /* ARMv7 defines bits for unimplemented coprocessors as RAZ/WI.
 546          * ASEDIS [31] and D32DIS [30] are both UNK/SBZP without VFP.
 547          * TRCDIS [28] is RAZ/WI since we do not implement a trace macrocell.
 548          */
 549         if (arm_feature(env, ARM_FEATURE_VFP)) {
 550             /* VFP coprocessor: cp10 & cp11 [23:20] */
 551             mask |= (1 << 31) | (1 << 30) | (0xf << 20);
 552
 553             if (!arm_feature(env, ARM_FEATURE_NEON)) {
 554                 /* ASEDIS [31] bit is RAO/WI */
 555                 value |= (1 << 31);
 556             }
 557
 558             /* VFPv3 and upwards with NEON implement 32 double precision
 559              * registers (D0-D31).
 560              */
 561             if (!arm_feature(env, ARM_FEATURE_NEON) ||
 562                     !arm_feature(env, ARM_FEATURE_VFP3)) {
 563                 /* D32DIS [30] is RAO/WI if D16-31 are not implemented. */
 564                 value |= (1 << 30);
 565             }
 566         }
 567         value &= mask;
 568     }
 569     env->cp15.c1_coproc = value;
 570 }
 571
 572 static const ARMCPRegInfo v6_cp_reginfo[] = {
 573     /* prefetch by MVA in v6, NOP in v7 */
 574     { .name = "MVA_prefetch",
 575       .cp = 15, .crn = 7, .crm = 13, .opc1 = 0, .opc2 = 1,
 576       .access = PL1_W, .type = ARM_CP_NOP },
 577     { .name = "ISB", .cp = 15, .crn = 7, .crm = 5, .opc1 = 0, .opc2 = 4,
 578       .access = PL0_W, .type = ARM_CP_NOP },
 579     { .name = "DSB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 4,
 580       .access = PL0_W, .type = ARM_CP_NOP },
 581     { .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
 582       .access = PL0_W, .type = ARM_CP_NOP },
 583     { .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
 584       .access = PL1_RW,
 585       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ifar_s),
 586                              offsetof(CPUARMState, cp15.ifar_ns) },
 587       .resetvalue = 0, },
 588     /* Watchpoint Fault Address Register : should actually only be present
 589      * for 1136, 1176, 11MPCore.
 590      */
 591     { .name = "WFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 1,
 592       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0, },
 593     { .name = "CPACR", .state = ARM_CP_STATE_BOTH, .opc0 = 3,
 594       .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 2,
 595       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c1_coproc),
 596       .resetvalue = 0, .writefn = cpacr_write },
 597     REGINFO_SENTINEL
 598 };
 599
 600 static CPAccessResult pmreg_access(CPUARMState *env, const ARMCPRegInfo *ri)
 601 {
 602     /* Performance monitor registers user accessibility is controlled
 603      * by PMUSERENR.
 604      */
 605     if (arm_current_el(env) == 0 && !env->cp15.c9_pmuserenr) {
 606         return CP_ACCESS_TRAP;
 607     }
 608     return CP_ACCESS_OK;
 609 }
 610
 611 #ifndef CONFIG_USER_ONLY
 612
 613 static inline bool arm_ccnt_enabled(CPUARMState *env)
 614 {
 615     /* This does not support checking PMCCFILTR_EL0 register */
 616
 617     if (!(env->cp15.c9_pmcr & PMCRE)) {
 618         return false;
 619     }
 620
 621     return true;
 622 }
 623
 624 void pmccntr_sync(CPUARMState *env)
 625 {
 626     uint64_t temp_ticks;
 627
 628     temp_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 629                           get_ticks_per_sec(), 1000000);
 630
 631     if (env->cp15.c9_pmcr & PMCRD) {
 632         /* Increment once every 64 processor clock cycles */
 633         temp_ticks /= 64;
 634     }
 635
 636     if (arm_ccnt_enabled(env)) {
 637         env->cp15.c15_ccnt = temp_ticks - env->cp15.c15_ccnt;
 638     }
 639 }
 640
 641 static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 642                        uint64_t value)
 643 {
 644     pmccntr_sync(env);
 645
 646     if (value & PMCRC) {
 647         /* The counter has been reset */
 648         env->cp15.c15_ccnt = 0;
 649     }
 650
 651     /* only the DP, X, D and E bits are writable */
 652     env->cp15.c9_pmcr &= ~0x39;
 653     env->cp15.c9_pmcr |= (value & 0x39);
 654
 655     pmccntr_sync(env);
 656 }
 657
 658 static uint64_t pmccntr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 659 {
 660     uint64_t total_ticks;
 661
 662     if (!arm_ccnt_enabled(env)) {
 663         /* Counter is disabled, do not change value */
 664         return env->cp15.c15_ccnt;
 665     }
 666
 667     total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 668                            get_ticks_per_sec(), 1000000);
 669
 670     if (env->cp15.c9_pmcr & PMCRD) {
 671         /* Increment once every 64 processor clock cycles */
 672         total_ticks /= 64;
 673     }
 674     return total_ticks - env->cp15.c15_ccnt;
 675 }
 676
 677 static void pmccntr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 678                         uint64_t value)
 679 {
 680     uint64_t total_ticks;
 681
 682     if (!arm_ccnt_enabled(env)) {
 683         /* Counter is disabled, set the absolute value */
 684         env->cp15.c15_ccnt = value;
 685         return;
 686     }
 687
 688     total_ticks = muldiv64(qemu_clock_get_us(QEMU_CLOCK_VIRTUAL),
 689                            get_ticks_per_sec(), 1000000);
 690
 691     if (env->cp15.c9_pmcr & PMCRD) {
 692         /* Increment once every 64 processor clock cycles */
 693         total_ticks /= 64;
 694     }
 695     env->cp15.c15_ccnt = total_ticks - value;
 696 }
 697
 698 static void pmccntr_write32(CPUARMState *env, const ARMCPRegInfo *ri,
 699                             uint64_t value)
 700 {
 701     uint64_t cur_val = pmccntr_read(env, NULL);
 702
 703     pmccntr_write(env, ri, deposit64(cur_val, 0, 32, value));
 704 }
 705
 706 #else /* CONFIG_USER_ONLY */
 707
 708 void pmccntr_sync(CPUARMState *env)
 709 {
 710 }
 711
 712 #endif
 713
 714 static void pmccfiltr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 715                             uint64_t value)
 716 {
 717     pmccntr_sync(env);
 718     env->cp15.pmccfiltr_el0 = value & 0x7E000000;
 719     pmccntr_sync(env);
 720 }
 721
 722 static void pmcntenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 723                             uint64_t value)
 724 {
 725     value &= (1 << 31);
 726     env->cp15.c9_pmcnten |= value;
 727 }
 728
 729 static void pmcntenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 730                              uint64_t value)
 731 {
 732     value &= (1 << 31);
 733     env->cp15.c9_pmcnten &= ~value;
 734 }
 735
 736 static void pmovsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 737                          uint64_t value)
 738 {
 739     env->cp15.c9_pmovsr &= ~value;
 740 }
 741
 742 static void pmxevtyper_write(CPUARMState *env, const ARMCPRegInfo *ri,
 743                              uint64_t value)
 744 {
 745     env->cp15.c9_pmxevtyper = value & 0xff;
 746 }
 747
 748 static void pmuserenr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 749                             uint64_t value)
 750 {
 751     env->cp15.c9_pmuserenr = value & 1;
 752 }
 753
 754 static void pmintenset_write(CPUARMState *env, const ARMCPRegInfo *ri,
 755                              uint64_t value)
 756 {
 757     /* We have no event counters so only the C bit can be changed */
 758     value &= (1 << 31);
 759     env->cp15.c9_pminten |= value;
 760 }
 761
 762 static void pmintenclr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 763                              uint64_t value)
 764 {
 765     value &= (1 << 31);
 766     env->cp15.c9_pminten &= ~value;
 767 }
 768
 769 static void vbar_write(CPUARMState *env, const ARMCPRegInfo *ri,
 770                        uint64_t value)
 771 {
 772     /* Note that even though the AArch64 view of this register has bits
 773      * [10:0] all RES0 we can only mask the bottom 5, to comply with the
 774      * architectural requirements for bits which are RES0 only in some
 775      * contexts. (ARMv8 would permit us to do no masking at all, but ARMv7
 776      * requires the bottom five bits to be RAZ/WI because they're UNK/SBZP.)
 777      */
 778     raw_write(env, ri, value & ~0x1FULL);
 779 }
 780
 781 static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 782 {
 783     /* We only mask off bits that are RES0 both for AArch64 and AArch32.
 784      * For bits that vary between AArch32/64, code needs to check the
 785      * current execution mode before directly using the feature bit.
 786      */
 787     uint32_t valid_mask = SCR_AARCH64_MASK | SCR_AARCH32_MASK;
 788
 789     if (!arm_feature(env, ARM_FEATURE_EL2)) {
 790         valid_mask &= ~SCR_HCE;
 791
 792         /* On ARMv7, SMD (or SCD as it is called in v7) is only
 793          * supported if EL2 exists. The bit is UNK/SBZP when
 794          * EL2 is unavailable. In QEMU ARMv7, we force it to always zero
 795          * when EL2 is unavailable.
 796          */
 797         if (arm_feature(env, ARM_FEATURE_V7)) {
 798             valid_mask &= ~SCR_SMD;
 799         }
 800     }
 801
 802     /* Clear all-context RES0 bits.  */
 803     value &= valid_mask;
 804     raw_write(env, ri, value);
 805 }
 806
 807 static uint64_t ccsidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 808 {
 809     ARMCPU *cpu = arm_env_get_cpu(env);
 810
 811     /* Acquire the CSSELR index from the bank corresponding to the CCSIDR
 812      * bank
 813      */
 814     uint32_t index = A32_BANKED_REG_GET(env, csselr,
 815                                         ri->secure & ARM_CP_SECSTATE_S);
 816
 817     return cpu->ccsidr[index];
 818 }
 819
 820 static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
 821                          uint64_t value)
 822 {
 823     raw_write(env, ri, value & 0xf);
 824 }
 825
 826 static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
 827 {
 828     CPUState *cs = ENV_GET_CPU(env);
 829     uint64_t ret = 0;
 830
 831     if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
 832         ret |= CPSR_I;
 833     }
 834     if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
 835         ret |= CPSR_F;
 836     }
 837     /* External aborts are not possible in QEMU so A bit is always clear */
 838     return ret;
 839 }
 840
 841 static const ARMCPRegInfo v7_cp_reginfo[] = {
 842     /* the old v6 WFI, UNPREDICTABLE in v7 but we choose to NOP */
 843     { .name = "NOP", .cp = 15, .crn = 7, .crm = 0, .opc1 = 0, .opc2 = 4,
 844       .access = PL1_W, .type = ARM_CP_NOP },
 845     /* Performance monitors are implementation defined in v7,
 846      * but with an ARM recommended set of registers, which we
 847      * follow (although we don't actually implement any counters)
 848      *
 849      * Performance registers fall into three categories:
 850      *  (a) always UNDEF in PL0, RW in PL1 (PMINTENSET, PMINTENCLR)
 851      *  (b) RO in PL0 (ie UNDEF on write), RW in PL1 (PMUSERENR)
 852      *  (c) UNDEF in PL0 if PMUSERENR.EN==0, otherwise accessible (all others)
 853      * For the cases controlled by PMUSERENR we must set .access to PL0_RW
 854      * or PL0_RO as appropriate and then check PMUSERENR in the helper fn.
 855      */
 856     { .name = "PMCNTENSET", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 1,
 857       .access = PL0_RW, .type = ARM_CP_NO_MIGRATE,
 858       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
 859       .writefn = pmcntenset_write,
 860       .accessfn = pmreg_access,
 861       .raw_writefn = raw_write },
 862     { .name = "PMCNTENSET_EL0", .state = ARM_CP_STATE_AA64,
 863       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 1,
 864       .access = PL0_RW, .accessfn = pmreg_access,
 865       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten), .resetvalue = 0,
 866       .writefn = pmcntenset_write, .raw_writefn = raw_write },
 867     { .name = "PMCNTENCLR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 2,
 868       .access = PL0_RW,
 869       .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcnten),
 870       .accessfn = pmreg_access,
 871       .writefn = pmcntenclr_write,
 872       .type = ARM_CP_NO_MIGRATE },
 873     { .name = "PMCNTENCLR_EL0", .state = ARM_CP_STATE_AA64,
 874       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 2,
 875       .access = PL0_RW, .accessfn = pmreg_access,
 876       .type = ARM_CP_NO_MIGRATE,
 877       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcnten),
 878       .writefn = pmcntenclr_write },
 879     { .name = "PMOVSR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 3,
 880       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, cp15.c9_pmovsr),
 881       .accessfn = pmreg_access,
 882       .writefn = pmovsr_write,
 883       .raw_writefn = raw_write },
 884     /* Unimplemented so WI. */
 885     { .name = "PMSWINC", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 4,
 886       .access = PL0_W, .accessfn = pmreg_access, .type = ARM_CP_NOP },
 887     /* Since we don't implement any events, writing to PMSELR is UNPREDICTABLE.
 888      * We choose to RAZ/WI.
 889      */
 890     { .name = "PMSELR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 5,
 891       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
 892       .accessfn = pmreg_access },
 893 #ifndef CONFIG_USER_ONLY
 894     { .name = "PMCCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 0,
 895       .access = PL0_RW, .resetvalue = 0, .type = ARM_CP_IO,
 896       .readfn = pmccntr_read, .writefn = pmccntr_write32,
 897       .accessfn = pmreg_access },
 898     { .name = "PMCCNTR_EL0", .state = ARM_CP_STATE_AA64,
 899       .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 13, .opc2 = 0,
 900       .access = PL0_RW, .accessfn = pmreg_access,
 901       .type = ARM_CP_IO,
 902       .readfn = pmccntr_read, .writefn = pmccntr_write, },
 903 #endif
 904     { .name = "PMCCFILTR_EL0", .state = ARM_CP_STATE_AA64,
 905       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 15, .opc2 = 7,
 906       .writefn = pmccfiltr_write,
 907       .access = PL0_RW, .accessfn = pmreg_access,
 908       .type = ARM_CP_IO,
 909       .fieldoffset = offsetof(CPUARMState, cp15.pmccfiltr_el0),
 910       .resetvalue = 0, },
 911     { .name = "PMXEVTYPER", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 1,
 912       .access = PL0_RW,
 913       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmxevtyper),
 914       .accessfn = pmreg_access, .writefn = pmxevtyper_write,
 915       .raw_writefn = raw_write },
 916     /* Unimplemented, RAZ/WI. */
 917     { .name = "PMXEVCNTR", .cp = 15, .crn = 9, .crm = 13, .opc1 = 0, .opc2 = 2,
 918       .access = PL0_RW, .type = ARM_CP_CONST, .resetvalue = 0,
 919       .accessfn = pmreg_access },
 920     { .name = "PMUSERENR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 0,
 921       .access = PL0_R | PL1_RW,
 922       .fieldoffset = offsetof(CPUARMState, cp15.c9_pmuserenr),
 923       .resetvalue = 0,
 924       .writefn = pmuserenr_write, .raw_writefn = raw_write },
 925     { .name = "PMINTENSET", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 1,
 926       .access = PL1_RW,
 927       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
 928       .resetvalue = 0,
 929       .writefn = pmintenset_write, .raw_writefn = raw_write },
 930     { .name = "PMINTENCLR", .cp = 15, .crn = 9, .crm = 14, .opc1 = 0, .opc2 = 2,
 931       .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
 932       .fieldoffset = offsetof(CPUARMState, cp15.c9_pminten),
 933       .resetvalue = 0, .writefn = pmintenclr_write, },
 934     { .name = "VBAR", .state = ARM_CP_STATE_BOTH,
 935       .opc0 = 3, .crn = 12, .crm = 0, .opc1 = 0, .opc2 = 0,
 936       .access = PL1_RW, .writefn = vbar_write,
 937       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.vbar_s),
 938                              offsetof(CPUARMState, cp15.vbar_ns) },
 939       .resetvalue = 0 },
 940     { .name = "CCSIDR", .state = ARM_CP_STATE_BOTH,
 941       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 0,
 942       .access = PL1_R, .readfn = ccsidr_read, .type = ARM_CP_NO_MIGRATE },
 943     { .name = "CSSELR", .state = ARM_CP_STATE_BOTH,
 944       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 2, .opc2 = 0,
 945       .access = PL1_RW, .writefn = csselr_write, .resetvalue = 0,
 946       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.csselr_s),
 947                              offsetof(CPUARMState, cp15.csselr_ns) } },
 948     /* Auxiliary ID register: this actually has an IMPDEF value but for now
 949      * just RAZ for all cores:
 950      */
 951     { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
 952       .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
 953       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
 954     /* Auxiliary fault status registers: these also are IMPDEF, and we
 955      * choose to RAZ/WI for all cores.
 956      */
 957     { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
 958       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
 959       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
 960     { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
 961       .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
 962       .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
 963     /* MAIR can just read-as-written because we don't implement caches
 964      * and so don't need to care about memory attributes.
 965      */
 966     { .name = "MAIR_EL1", .state = ARM_CP_STATE_AA64,
 967       .opc0 = 3, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0,
 968       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.mair_el[1]),
 969       .resetvalue = 0 },
 970     /* For non-long-descriptor page tables these are PRRR and NMRR;
 971      * regardless they still act as reads-as-written for QEMU.
 972      * The override is necessary because of the overly-broad TLB_LOCKDOWN
 973      * definition.
 974      */
 975      /* MAIR0/1 are defined separately from their 64-bit counterpart which
 976       * allows them to assign the correct fieldoffset based on the endianness
 977       * handled in the field definitions.
 978       */
 979     { .name = "MAIR0", .state = ARM_CP_STATE_AA32, .type = ARM_CP_OVERRIDE,
 980       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 0, .access = PL1_RW,
 981       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair0_s),
 982                              offsetof(CPUARMState, cp15.mair0_ns) },
 983       .resetfn = arm_cp_reset_ignore },
 984     { .name = "MAIR1", .state = ARM_CP_STATE_AA32, .type = ARM_CP_OVERRIDE,
 985       .cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1, .access = PL1_RW,
 986       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.mair1_s),
 987                              offsetof(CPUARMState, cp15.mair1_ns) },
 988       .resetfn = arm_cp_reset_ignore },
 989     { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
 990       .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
 991       .type = ARM_CP_NO_MIGRATE, .access = PL1_R, .readfn = isr_read },
 992     /* 32 bit ITLB invalidates */
 993     { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
 994       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
 995     { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
 996       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
 997     { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
 998       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
 999     /* 32 bit DTLB invalidates */
1000     { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
1001       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
1002     { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
1003       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
1004     { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
1005       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
1006     /* 32 bit TLB invalidates */
1007     { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
1008       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
1009     { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
1010       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
1011     { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
1012       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
1013     { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
1014       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
1015     REGINFO_SENTINEL
1016 };
1017
1018 static const ARMCPRegInfo v7mp_cp_reginfo[] = {
1019     /* 32 bit TLB invalidates, Inner Shareable */
1020     { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
1021       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_is_write },
1022     { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
1023       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_is_write },
1024     { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
1025       .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
1026       .writefn = tlbiasid_is_write },
1027     { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
1028       .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
1029       .writefn = tlbimvaa_is_write },
1030     REGINFO_SENTINEL
1031 };
1032
1033 static void teecr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1034                         uint64_t value)
1035 {
1036     value &= 1;
1037     env->teecr = value;
1038 }
1039
1040 static CPAccessResult teehbr_access(CPUARMState *env, const ARMCPRegInfo *ri)
1041 {
1042     if (arm_current_el(env) == 0 && (env->teecr & 1)) {
1043         return CP_ACCESS_TRAP;
1044     }
1045     return CP_ACCESS_OK;
1046 }
1047
1048 static const ARMCPRegInfo t2ee_cp_reginfo[] = {
1049     { .name = "TEECR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 6, .opc2 = 0,
1050       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, teecr),
1051       .resetvalue = 0,
1052       .writefn = teecr_write },
1053     { .name = "TEEHBR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 6, .opc2 = 0,
1054       .access = PL0_RW, .fieldoffset = offsetof(CPUARMState, teehbr),
1055       .accessfn = teehbr_access, .resetvalue = 0 },
1056     REGINFO_SENTINEL
1057 };
1058
1059 static const ARMCPRegInfo v6k_cp_reginfo[] = {
1060     { .name = "TPIDR_EL0", .state = ARM_CP_STATE_AA64,
1061       .opc0 = 3, .opc1 = 3, .opc2 = 2, .crn = 13, .crm = 0,
1062       .access = PL0_RW,
1063       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[0]), .resetvalue = 0 },
1064     { .name = "TPIDRURW", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 2,
1065       .access = PL0_RW,
1066       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrurw_s),
1067                              offsetoflow32(CPUARMState, cp15.tpidrurw_ns) },
1068       .resetfn = arm_cp_reset_ignore },
1069     { .name = "TPIDRRO_EL0", .state = ARM_CP_STATE_AA64,
1070       .opc0 = 3, .opc1 = 3, .opc2 = 3, .crn = 13, .crm = 0,
1071       .access = PL0_R|PL1_W,
1072       .fieldoffset = offsetof(CPUARMState, cp15.tpidrro_el[0]),
1073       .resetvalue = 0},
1074     { .name = "TPIDRURO", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 3,
1075       .access = PL0_R|PL1_W,
1076       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidruro_s),
1077                              offsetoflow32(CPUARMState, cp15.tpidruro_ns) },
1078       .resetfn = arm_cp_reset_ignore },
1079     { .name = "TPIDR_EL1", .state = ARM_CP_STATE_AA64,
1080       .opc0 = 3, .opc1 = 0, .opc2 = 4, .crn = 13, .crm = 0,
1081       .access = PL1_RW,
1082       .fieldoffset = offsetof(CPUARMState, cp15.tpidr_el[1]), .resetvalue = 0 },
1083     { .name = "TPIDRPRW", .opc1 = 0, .cp = 15, .crn = 13, .crm = 0, .opc2 = 4,
1084       .access = PL1_RW,
1085       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tpidrprw_s),
1086                              offsetoflow32(CPUARMState, cp15.tpidrprw_ns) },
1087       .resetvalue = 0 },
1088     REGINFO_SENTINEL
1089 };
1090
1091 #ifndef CONFIG_USER_ONLY
1092
1093 static CPAccessResult gt_cntfrq_access(CPUARMState *env, const ARMCPRegInfo *ri)
1094 {
1095     /* CNTFRQ: not visible from PL0 if both PL0PCTEN and PL0VCTEN are zero */
1096     if (arm_current_el(env) == 0 && !extract32(env->cp15.c14_cntkctl, 0, 2)) {
1097         return CP_ACCESS_TRAP;
1098     }
1099     return CP_ACCESS_OK;
1100 }
1101
1102 static CPAccessResult gt_counter_access(CPUARMState *env, int timeridx)
1103 {
1104     /* CNT[PV]CT: not visible from PL0 if ELO[PV]CTEN is zero */
1105     if (arm_current_el(env) == 0 &&
1106         !extract32(env->cp15.c14_cntkctl, timeridx, 1)) {
1107         return CP_ACCESS_TRAP;
1108     }
1109     return CP_ACCESS_OK;
1110 }
1111
1112 static CPAccessResult gt_timer_access(CPUARMState *env, int timeridx)
1113 {
1114     /* CNT[PV]_CVAL, CNT[PV]_CTL, CNT[PV]_TVAL: not visible from PL0 if
1115      * EL0[PV]TEN is zero.
1116      */
1117     if (arm_current_el(env) == 0 &&
1118         !extract32(env->cp15.c14_cntkctl, 9 - timeridx, 1)) {
1119         return CP_ACCESS_TRAP;
1120     }
1121     return CP_ACCESS_OK;
1122 }
1123
1124 static CPAccessResult gt_pct_access(CPUARMState *env,
1125                                          const ARMCPRegInfo *ri)
1126 {
1127     return gt_counter_access(env, GTIMER_PHYS);
1128 }
1129
1130 static CPAccessResult gt_vct_access(CPUARMState *env,
1131                                          const ARMCPRegInfo *ri)
1132 {
1133     return gt_counter_access(env, GTIMER_VIRT);
1134 }
1135
1136 static CPAccessResult gt_ptimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
1137 {
1138     return gt_timer_access(env, GTIMER_PHYS);
1139 }
1140
1141 static CPAccessResult gt_vtimer_access(CPUARMState *env, const ARMCPRegInfo *ri)
1142 {
1143     return gt_timer_access(env, GTIMER_VIRT);
1144 }
1145
1146 static uint64_t gt_get_countervalue(CPUARMState *env)
1147 {
1148     return qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) / GTIMER_SCALE;
1149 }
1150
1151 static void gt_recalc_timer(ARMCPU *cpu, int timeridx)
1152 {
1153     ARMGenericTimer *gt = &cpu->env.cp15.c14_timer[timeridx];
1154
1155     if (gt->ctl & 1) {
1156         /* Timer enabled: calculate and set current ISTATUS, irq, and
1157          * reset timer to when ISTATUS next has to change
1158          */
1159         uint64_t count = gt_get_countervalue(&cpu->env);
1160         /* Note that this must be unsigned 64 bit arithmetic: */
1161         int istatus = count >= gt->cval;
1162         uint64_t nexttick;
1163
1164         gt->ctl = deposit32(gt->ctl, 2, 1, istatus);
1165         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1166                      (istatus && !(gt->ctl & 2)));
1167         if (istatus) {
1168             /* Next transition is when count rolls back over to zero */
1169             nexttick = UINT64_MAX;
1170         } else {
1171             /* Next transition is when we hit cval */
1172             nexttick = gt->cval;
1173         }
1174         /* Note that the desired next expiry time might be beyond the
1175          * signed-64-bit range of a QEMUTimer -- in this case we just
1176          * set the timer for as far in the future as possible. When the
1177          * timer expires we will reset the timer for any remaining period.
1178          */
1179         if (nexttick > INT64_MAX / GTIMER_SCALE) {
1180             nexttick = INT64_MAX / GTIMER_SCALE;
1181         }
1182         timer_mod(cpu->gt_timer[timeridx], nexttick);
1183     } else {
1184         /* Timer disabled: ISTATUS and timer output always clear */
1185         gt->ctl &= ~4;
1186         qemu_set_irq(cpu->gt_timer_outputs[timeridx], 0);
1187         timer_del(cpu->gt_timer[timeridx]);
1188     }
1189 }
1190
1191 static void gt_cnt_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1192 {
1193     ARMCPU *cpu = arm_env_get_cpu(env);
1194     int timeridx = ri->opc1 & 1;
1195
1196     timer_del(cpu->gt_timer[timeridx]);
1197 }
1198
1199 static uint64_t gt_cnt_read(CPUARMState *env, const ARMCPRegInfo *ri)
1200 {
1201     return gt_get_countervalue(env);
1202 }
1203
1204 static void gt_cval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1205                           uint64_t value)
1206 {
1207     int timeridx = ri->opc1 & 1;
1208
1209     env->cp15.c14_timer[timeridx].cval = value;
1210     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1211 }
1212
1213 static uint64_t gt_tval_read(CPUARMState *env, const ARMCPRegInfo *ri)
1214 {
1215     int timeridx = ri->crm & 1;
1216
1217     return (uint32_t)(env->cp15.c14_timer[timeridx].cval -
1218                       gt_get_countervalue(env));
1219 }
1220
1221 static void gt_tval_write(CPUARMState *env, const ARMCPRegInfo *ri,
1222                           uint64_t value)
1223 {
1224     int timeridx = ri->crm & 1;
1225
1226     env->cp15.c14_timer[timeridx].cval = gt_get_countervalue(env) +
1227         + sextract64(value, 0, 32);
1228     gt_recalc_timer(arm_env_get_cpu(env), timeridx);
1229 }
1230
1231 static void gt_ctl_write(CPUARMState *env, const ARMCPRegInfo *ri,
1232                          uint64_t value)
1233 {
1234     ARMCPU *cpu = arm_env_get_cpu(env);
1235     int timeridx = ri->crm & 1;
1236     uint32_t oldval = env->cp15.c14_timer[timeridx].ctl;
1237
1238     env->cp15.c14_timer[timeridx].ctl = deposit64(oldval, 0, 2, value);
1239     if ((oldval ^ value) & 1) {
1240         /* Enable toggled */
1241         gt_recalc_timer(cpu, timeridx);
1242     } else if ((oldval ^ value) & 2) {
1243         /* IMASK toggled: don't need to recalculate,
1244          * just set the interrupt line based on ISTATUS
1245          */
1246         qemu_set_irq(cpu->gt_timer_outputs[timeridx],
1247                      (oldval & 4) && !(value & 2));
1248     }
1249 }
1250
1251 void arm_gt_ptimer_cb(void *opaque)
1252 {
1253     ARMCPU *cpu = opaque;
1254
1255     gt_recalc_timer(cpu, GTIMER_PHYS);
1256 }
1257
1258 void arm_gt_vtimer_cb(void *opaque)
1259 {
1260     ARMCPU *cpu = opaque;
1261
1262     gt_recalc_timer(cpu, GTIMER_VIRT);
1263 }
1264
1265 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1266     /* Note that CNTFRQ is purely reads-as-written for the benefit
1267      * of software; writing it doesn't actually change the timer frequency.
1268      * Our reset value matches the fixed frequency we implement the timer at.
1269      */
1270     { .name = "CNTFRQ", .cp = 15, .crn = 14, .crm = 0, .opc1 = 0, .opc2 = 0,
1271       .type = ARM_CP_NO_MIGRATE,
1272       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1273       .fieldoffset = offsetoflow32(CPUARMState, cp15.c14_cntfrq),
1274       .resetfn = arm_cp_reset_ignore,
1275     },
1276     { .name = "CNTFRQ_EL0", .state = ARM_CP_STATE_AA64,
1277       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 0,
1278       .access = PL1_RW | PL0_R, .accessfn = gt_cntfrq_access,
1279       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntfrq),
1280       .resetvalue = (1000 * 1000 * 1000) / GTIMER_SCALE,
1281     },
1282     /* overall control: mostly access permissions */
1283     { .name = "CNTKCTL", .state = ARM_CP_STATE_BOTH,
1284       .opc0 = 3, .opc1 = 0, .crn = 14, .crm = 1, .opc2 = 0,
1285       .access = PL1_RW,
1286       .fieldoffset = offsetof(CPUARMState, cp15.c14_cntkctl),
1287       .resetvalue = 0,
1288     },
1289     /* per-timer control */
1290     { .name = "CNTP_CTL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 1,
1291       .type = ARM_CP_IO | ARM_CP_NO_MIGRATE, .access = PL1_RW | PL0_R,
1292       .accessfn = gt_ptimer_access,
1293       .fieldoffset = offsetoflow32(CPUARMState,
1294                                    cp15.c14_timer[GTIMER_PHYS].ctl),
1295       .resetfn = arm_cp_reset_ignore,
1296       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1297     },
1298     { .name = "CNTP_CTL_EL0", .state = ARM_CP_STATE_AA64,
1299       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 1,
1300       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1301       .accessfn = gt_ptimer_access,
1302       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].ctl),
1303       .resetvalue = 0,
1304       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1305     },
1306     { .name = "CNTV_CTL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 1,
1307       .type = ARM_CP_IO | ARM_CP_NO_MIGRATE, .access = PL1_RW | PL0_R,
1308       .accessfn = gt_vtimer_access,
1309       .fieldoffset = offsetoflow32(CPUARMState,
1310                                    cp15.c14_timer[GTIMER_VIRT].ctl),
1311       .resetfn = arm_cp_reset_ignore,
1312       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1313     },
1314     { .name = "CNTV_CTL_EL0", .state = ARM_CP_STATE_AA64,
1315       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 1,
1316       .type = ARM_CP_IO, .access = PL1_RW | PL0_R,
1317       .accessfn = gt_vtimer_access,
1318       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].ctl),
1319       .resetvalue = 0,
1320       .writefn = gt_ctl_write, .raw_writefn = raw_write,
1321     },
1322     /* TimerValue views: a 32 bit downcounting view of the underlying state */
1323     { .name = "CNTP_TVAL", .cp = 15, .crn = 14, .crm = 2, .opc1 = 0, .opc2 = 0,
1324       .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
1325       .accessfn = gt_ptimer_access,
1326       .readfn = gt_tval_read, .writefn = gt_tval_write,
1327     },
1328     { .name = "CNTP_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1329       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 0,
1330       .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
1331       .readfn = gt_tval_read, .writefn = gt_tval_write,
1332     },
1333     { .name = "CNTV_TVAL", .cp = 15, .crn = 14, .crm = 3, .opc1 = 0, .opc2 = 0,
1334       .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
1335       .accessfn = gt_vtimer_access,
1336       .readfn = gt_tval_read, .writefn = gt_tval_write,
1337     },
1338     { .name = "CNTV_TVAL_EL0", .state = ARM_CP_STATE_AA64,
1339       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 0,
1340       .type = ARM_CP_NO_MIGRATE | ARM_CP_IO, .access = PL1_RW | PL0_R,
1341       .readfn = gt_tval_read, .writefn = gt_tval_write,
1342     },
1343     /* The counter itself */
1344     { .name = "CNTPCT", .cp = 15, .crm = 14, .opc1 = 0,
1345       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE | ARM_CP_IO,
1346       .accessfn = gt_pct_access,
1347       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1348     },
1349     { .name = "CNTPCT_EL0", .state = ARM_CP_STATE_AA64,
1350       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 1,
1351       .access = PL0_R, .type = ARM_CP_NO_MIGRATE | ARM_CP_IO,
1352       .accessfn = gt_pct_access,
1353       .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
1354     },
1355     { .name = "CNTVCT", .cp = 15, .crm = 14, .opc1 = 1,
1356       .access = PL0_R, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE | ARM_CP_IO,
1357       .accessfn = gt_vct_access,
1358       .readfn = gt_cnt_read, .resetfn = arm_cp_reset_ignore,
1359     },
1360     { .name = "CNTVCT_EL0", .state = ARM_CP_STATE_AA64,
1361       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 0, .opc2 = 2,
1362       .access = PL0_R, .type = ARM_CP_NO_MIGRATE | ARM_CP_IO,
1363       .accessfn = gt_vct_access,
1364       .readfn = gt_cnt_read, .resetfn = gt_cnt_reset,
1365     },
1366     /* Comparison value, indicating when the timer goes off */
1367     { .name = "CNTP_CVAL", .cp = 15, .crm = 14, .opc1 = 2,
1368       .access = PL1_RW | PL0_R,
1369       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_NO_MIGRATE,
1370       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1371       .accessfn = gt_ptimer_access, .resetfn = arm_cp_reset_ignore,
1372       .writefn = gt_cval_write, .raw_writefn = raw_write,
1373     },
1374     { .name = "CNTP_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1375       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 2, .opc2 = 2,
1376       .access = PL1_RW | PL0_R,
1377       .type = ARM_CP_IO,
1378       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_PHYS].cval),
1379       .resetvalue = 0, .accessfn = gt_vtimer_access,
1380       .writefn = gt_cval_write, .raw_writefn = raw_write,
1381     },
1382     { .name = "CNTV_CVAL", .cp = 15, .crm = 14, .opc1 = 3,
1383       .access = PL1_RW | PL0_R,
1384       .type = ARM_CP_64BIT | ARM_CP_IO | ARM_CP_NO_MIGRATE,
1385       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1386       .accessfn = gt_vtimer_access, .resetfn = arm_cp_reset_ignore,
1387       .writefn = gt_cval_write, .raw_writefn = raw_write,
1388     },
1389     { .name = "CNTV_CVAL_EL0", .state = ARM_CP_STATE_AA64,
1390       .opc0 = 3, .opc1 = 3, .crn = 14, .crm = 3, .opc2 = 2,
1391       .access = PL1_RW | PL0_R,
1392       .type = ARM_CP_IO,
1393       .fieldoffset = offsetof(CPUARMState, cp15.c14_timer[GTIMER_VIRT].cval),
1394       .resetvalue = 0, .accessfn = gt_vtimer_access,
1395       .writefn = gt_cval_write, .raw_writefn = raw_write,
1396     },
1397     REGINFO_SENTINEL
1398 };
1399
1400 #else
1401 /* In user-mode none of the generic timer registers are accessible,
1402  * and their implementation depends on QEMU_CLOCK_VIRTUAL and qdev gpio outputs,
1403  * so instead just don't register any of them.
1404  */
1405 static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
1406     REGINFO_SENTINEL
1407 };
1408
1409 #endif
1410
1411 static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1412 {
1413     if (arm_feature(env, ARM_FEATURE_LPAE)) {
1414         raw_write(env, ri, value);
1415     } else if (arm_feature(env, ARM_FEATURE_V7)) {
1416         raw_write(env, ri, value & 0xfffff6ff);
1417     } else {
1418         raw_write(env, ri, value & 0xfffff1ff);
1419     }
1420 }
1421
1422 #ifndef CONFIG_USER_ONLY
1423 /* get_phys_addr() isn't present for user-mode-only targets */
1424
1425 static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri)
1426 {
1427     if (ri->opc2 & 4) {
1428         /* Other states are only available with TrustZone; in
1429          * a non-TZ implementation these registers don't exist
1430          * at all, which is an Uncategorized trap. This underdecoding
1431          * is safe because the reginfo is NO_MIGRATE.
1432          */
1433         return CP_ACCESS_TRAP_UNCATEGORIZED;
1434     }
1435     return CP_ACCESS_OK;
1436 }
1437
1438 static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
1439 {
1440     hwaddr phys_addr;
1441     target_ulong page_size;
1442     int prot;
1443     int ret, is_user = ri->opc2 & 2;
1444     int access_type = ri->opc2 & 1;
1445     uint64_t par64;
1446
1447     ret = get_phys_addr(env, value, access_type, is_user,
1448                         &phys_addr, &prot, &page_size);
1449     if (extended_addresses_enabled(env)) {
1450         /* ret is a DFSR/IFSR value for the long descriptor
1451          * translation table format, but with WnR always clear.
1452          * Convert it to a 64-bit PAR.
1453          */
1454         par64 = (1 << 11); /* LPAE bit always set */
1455         if (ret == 0) {
1456             par64 |= phys_addr & ~0xfffULL;
1457             /* We don't set the ATTR or SH fields in the PAR. */
1458         } else {
1459             par64 |= 1; /* F */
1460             par64 |= (ret & 0x3f) << 1; /* FS */
1461             /* Note that S2WLK and FSTAGE are always zero, because we don't
1462              * implement virtualization and therefore there can't be a stage 2
1463              * fault.
1464              */
1465         }
1466     } else {
1467         /* ret is a DFSR/IFSR value for the short descriptor
1468          * translation table format (with WnR always clear).
1469          * Convert it to a 32-bit PAR.
1470          */
1471         if (ret == 0) {
1472             /* We do not set any attribute bits in the PAR */
1473             if (page_size == (1 << 24)
1474                 && arm_feature(env, ARM_FEATURE_V7)) {
1475                 par64 = (phys_addr & 0xff000000) | (1 << 1);
1476             } else {
1477                 par64 = phys_addr & 0xfffff000;
1478             }
1479         } else {
1480             par64 = ((ret & (1 << 10)) >> 5) | ((ret & (1 << 12)) >> 6) |
1481                     ((ret & 0xf) << 1) | 1;
1482         }
1483     }
1484
1485     A32_BANKED_CURRENT_REG_SET(env, par, par64);
1486 }
1487 #endif
1488
1489 static const ARMCPRegInfo vapa_cp_reginfo[] = {
1490     { .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
1491       .access = PL1_RW, .resetvalue = 0,
1492       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.par_s),
1493                              offsetoflow32(CPUARMState, cp15.par_ns) },
1494       .writefn = par_write },
1495 #ifndef CONFIG_USER_ONLY
1496     { .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
1497       .access = PL1_W, .accessfn = ats_access,
1498       .writefn = ats_write, .type = ARM_CP_NO_MIGRATE },
1499 #endif
1500     REGINFO_SENTINEL
1501 };
1502
1503 /* Return basic MPU access permission bits.  */
1504 static uint32_t simple_mpu_ap_bits(uint32_t val)
1505 {
1506     uint32_t ret;
1507     uint32_t mask;
1508     int i;
1509     ret = 0;
1510     mask = 3;
1511     for (i = 0; i < 16; i += 2) {
1512         ret |= (val >> i) & mask;
1513         mask <<= 2;
1514     }
1515     return ret;
1516 }
1517
1518 /* Pad basic MPU access permission bits to extended format.  */
1519 static uint32_t extended_mpu_ap_bits(uint32_t val)
1520 {
1521     uint32_t ret;
1522     uint32_t mask;
1523     int i;
1524     ret = 0;
1525     mask = 3;
1526     for (i = 0; i < 16; i += 2) {
1527         ret |= (val & mask) << i;
1528         mask <<= 2;
1529     }
1530     return ret;
1531 }
1532
1533 static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
1534                                  uint64_t value)
1535 {
1536     env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
1537 }
1538
1539 static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
1540 {
1541     return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
1542 }
1543
1544 static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
1545                                  uint64_t value)
1546 {
1547     env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
1548 }
1549
1550 static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
1551 {
1552     return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
1553 }
1554
1555 static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
1556     { .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
1557       .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
1558       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
1559       .resetvalue = 0,
1560       .readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
1561     { .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
1562       .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
1563       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
1564       .resetvalue = 0,
1565       .readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
1566     { .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
1567       .access = PL1_RW,
1568       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
1569       .resetvalue = 0, },
1570     { .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
1571       .access = PL1_RW,
1572       .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
1573       .resetvalue = 0, },
1574     { .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
1575       .access = PL1_RW,
1576       .fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
1577     { .name = "ICACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 1,
1578       .access = PL1_RW,
1579       .fieldoffset = offsetof(CPUARMState, cp15.c2_insn), .resetvalue = 0, },
1580     /* Protection region base and size registers */
1581     { .name = "946_PRBS0", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0,
1582       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1583       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[0]) },
1584     { .name = "946_PRBS1", .cp = 15, .crn = 6, .crm = 1, .opc1 = 0,
1585       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1586       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[1]) },
1587     { .name = "946_PRBS2", .cp = 15, .crn = 6, .crm = 2, .opc1 = 0,
1588       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1589       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[2]) },
1590     { .name = "946_PRBS3", .cp = 15, .crn = 6, .crm = 3, .opc1 = 0,
1591       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1592       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[3]) },
1593     { .name = "946_PRBS4", .cp = 15, .crn = 6, .crm = 4, .opc1 = 0,
1594       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1595       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[4]) },
1596     { .name = "946_PRBS5", .cp = 15, .crn = 6, .crm = 5, .opc1 = 0,
1597       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1598       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[5]) },
1599     { .name = "946_PRBS6", .cp = 15, .crn = 6, .crm = 6, .opc1 = 0,
1600       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1601       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[6]) },
1602     { .name = "946_PRBS7", .cp = 15, .crn = 6, .crm = 7, .opc1 = 0,
1603       .opc2 = CP_ANY, .access = PL1_RW, .resetvalue = 0,
1604       .fieldoffset = offsetof(CPUARMState, cp15.c6_region[7]) },
1605     REGINFO_SENTINEL
1606 };
1607
1608 static void vmsa_ttbcr_raw_write(CPUARMState *env, const ARMCPRegInfo *ri,
1609                                  uint64_t value)
1610 {
1611     TCR *tcr = raw_ptr(env, ri);
1612     int maskshift = extract32(value, 0, 3);
1613
1614     if (!arm_feature(env, ARM_FEATURE_V8)) {
1615         if (arm_feature(env, ARM_FEATURE_LPAE) && (value & TTBCR_EAE)) {
1616             /* Pre ARMv8 bits [21:19], [15:14] and [6:3] are UNK/SBZP when
1617              * using Long-desciptor translation table format */
1618             value &= ~((7 << 19) | (3 << 14) | (0xf << 3));
1619         } else if (arm_feature(env, ARM_FEATURE_EL3)) {
1620             /* In an implementation that includes the Security Extensions
1621              * TTBCR has additional fields PD0 [4] and PD1 [5] for
1622              * Short-descriptor translation table format.
1623              */
1624             value &= TTBCR_PD1 | TTBCR_PD0 | TTBCR_N;
1625         } else {
1626             value &= TTBCR_N;
1627         }
1628     }
1629
1630     /* Update the masks corresponding to the the TCR bank being written
1631      * Note that we always calculate mask and base_mask, but
1632      * they are only used for short-descriptor tables (ie if EAE is 0);
1633      * for long-descriptor tables the TCR fields are used differently
1634      * and the mask and base_mask values are meaningless.
1635      */
1636     tcr->raw_tcr = value;
1637     tcr->mask = ~(((uint32_t)0xffffffffu) >> maskshift);
1638     tcr->base_mask = ~((uint32_t)0x3fffu >> maskshift);
1639 }
1640
1641 static void vmsa_ttbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1642                              uint64_t value)
1643 {
1644     ARMCPU *cpu = arm_env_get_cpu(env);
1645
1646     if (arm_feature(env, ARM_FEATURE_LPAE)) {
1647         /* With LPAE the TTBCR could result in a change of ASID
1648          * via the TTBCR.A1 bit, so do a TLB flush.
1649          */
1650         tlb_flush(CPU(cpu), 1);
1651     }
1652     vmsa_ttbcr_raw_write(env, ri, value);
1653 }
1654
1655 static void vmsa_ttbcr_reset(CPUARMState *env, const ARMCPRegInfo *ri)
1656 {
1657     TCR *tcr = raw_ptr(env, ri);
1658
1659     /* Reset both the TCR as well as the masks corresponding to the bank of
1660      * the TCR being reset.
1661      */
1662     tcr->raw_tcr = 0;
1663     tcr->mask = 0;
1664     tcr->base_mask = 0xffffc000u;
1665 }
1666
1667 static void vmsa_tcr_el1_write(CPUARMState *env, const ARMCPRegInfo *ri,
1668                                uint64_t value)
1669 {
1670     ARMCPU *cpu = arm_env_get_cpu(env);
1671     TCR *tcr = raw_ptr(env, ri);
1672
1673     /* For AArch64 the A1 bit could result in a change of ASID, so TLB flush. */
1674     tlb_flush(CPU(cpu), 1);
1675     tcr->raw_tcr = value;
1676 }
1677
1678 static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1679                             uint64_t value)
1680 {
1681     /* 64 bit accesses to the TTBRs can change the ASID and so we
1682      * must flush the TLB.
1683      */
1684     if (cpreg_field_is_64bit(ri)) {
1685         ARMCPU *cpu = arm_env_get_cpu(env);
1686
1687         tlb_flush(CPU(cpu), 1);
1688     }
1689     raw_write(env, ri, value);
1690 }
1691
1692 static const ARMCPRegInfo vmsa_cp_reginfo[] = {
1693     { .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
1694       .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
1695       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dfsr_s),
1696                              offsetoflow32(CPUARMState, cp15.dfsr_ns) },
1697       .resetfn = arm_cp_reset_ignore, },
1698     { .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
1699       .access = PL1_RW, .resetvalue = 0,
1700       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.ifsr_s),
1701                              offsetoflow32(CPUARMState, cp15.ifsr_ns) } },
1702     { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
1703       .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
1704       .access = PL1_RW,
1705       .fieldoffset = offsetof(CPUARMState, cp15.esr_el[1]), .resetvalue = 0, },
1706     { .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
1707       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 0,
1708       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
1709       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
1710                              offsetof(CPUARMState, cp15.ttbr0_ns) } },
1711     { .name = "TTBR1_EL1", .state = ARM_CP_STATE_BOTH,
1712       .opc0 = 3, .opc1 = 0, .crn = 2, .crm = 0, .opc2 = 1,
1713       .access = PL1_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
1714       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
1715                              offsetof(CPUARMState, cp15.ttbr1_ns) } },
1716     { .name = "TCR_EL1", .state = ARM_CP_STATE_AA64,
1717       .opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
1718       .access = PL1_RW, .writefn = vmsa_tcr_el1_write,
1719       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
1720       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[1]) },
1721     { .name = "TTBCR", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 2,
1722       .access = PL1_RW, .type = ARM_CP_NO_MIGRATE, .writefn = vmsa_ttbcr_write,
1723       .resetfn = arm_cp_reset_ignore, .raw_writefn = vmsa_ttbcr_raw_write,
1724       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.tcr_el[3]),
1725                              offsetoflow32(CPUARMState, cp15.tcr_el[1])} },
1726     { .name = "FAR_EL1", .state = ARM_CP_STATE_AA64,
1727       .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
1728       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[1]),
1729       .resetvalue = 0, },
1730     { .name = "DFAR", .cp = 15, .opc1 = 0, .crn = 6, .crm = 0, .opc2 = 0,
1731       .access = PL1_RW, .resetvalue = 0,
1732       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.dfar_s),
1733                              offsetof(CPUARMState, cp15.dfar_ns) } },
1734     REGINFO_SENTINEL
1735 };
1736
1737 static void omap_ticonfig_write(CPUARMState *env, const ARMCPRegInfo *ri,
1738                                 uint64_t value)
1739 {
1740     env->cp15.c15_ticonfig = value & 0xe7;
1741     /* The OS_TYPE bit in this register changes the reported CPUID! */
1742     env->cp15.c0_cpuid = (value & (1 << 5)) ?
1743         ARM_CPUID_TI915T : ARM_CPUID_TI925T;
1744 }
1745
1746 static void omap_threadid_write(CPUARMState *env, const ARMCPRegInfo *ri,
1747                                 uint64_t value)
1748 {
1749     env->cp15.c15_threadid = value & 0xffff;
1750 }
1751
1752 static void omap_wfi_write(CPUARMState *env, const ARMCPRegInfo *ri,
1753                            uint64_t value)
1754 {
1755     /* Wait-for-interrupt (deprecated) */
1756     cpu_interrupt(CPU(arm_env_get_cpu(env)), CPU_INTERRUPT_HALT);
1757 }
1758
1759 static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
1760                                   uint64_t value)
1761 {
1762     /* On OMAP there are registers indicating the max/min index of dcache lines
1763      * containing a dirty line; cache flush operations have to reset these.
1764      */
1765     env->cp15.c15_i_max = 0x000;
1766     env->cp15.c15_i_min = 0xff0;
1767 }
1768
1769 static const ARMCPRegInfo omap_cp_reginfo[] = {
1770     { .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
1771       .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
1772       .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el[1]),
1773       .resetvalue = 0, },
1774     { .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
1775       .access = PL1_RW, .type = ARM_CP_NOP },
1776     { .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
1777       .access = PL1_RW,
1778       .fieldoffset = offsetof(CPUARMState, cp15.c15_ticonfig), .resetvalue = 0,
1779       .writefn = omap_ticonfig_write },
1780     { .name = "IMAX", .cp = 15, .crn = 15, .crm = 2, .opc1 = 0, .opc2 = 0,
1781       .access = PL1_RW,
1782       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_max), .resetvalue = 0, },
1783     { .name = "IMIN", .cp = 15, .crn = 15, .crm = 3, .opc1 = 0, .opc2 = 0,
1784       .access = PL1_RW, .resetvalue = 0xff0,
1785       .fieldoffset = offsetof(CPUARMState, cp15.c15_i_min) },
1786     { .name = "THREADID", .cp = 15, .crn = 15, .crm = 4, .opc1 = 0, .opc2 = 0,
1787       .access = PL1_RW,
1788       .fieldoffset = offsetof(CPUARMState, cp15.c15_threadid), .resetvalue = 0,
1789       .writefn = omap_threadid_write },
1790     { .name = "TI925T_STATUS", .cp = 15, .crn = 15,
1791       .crm = 8, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
1792       .type = ARM_CP_NO_MIGRATE,
1793       .readfn = arm_cp_read_zero, .writefn = omap_wfi_write, },
1794     /* TODO: Peripheral port remap register:
1795      * On OMAP2 mcr p15, 0, rn, c15, c2, 4 sets up the interrupt controller
1796      * base address at $rn & ~0xfff and map size of 0x200 << ($rn & 0xfff),
1797      * when MMU is off.
1798      */
1799     { .name = "OMAP_CACHEMAINT", .cp = 15, .crn = 7, .crm = CP_ANY,
1800       .opc1 = 0, .opc2 = CP_ANY, .access = PL1_W,
1801       .type = ARM_CP_OVERRIDE | ARM_CP_NO_MIGRATE,
1802       .writefn = omap_cachemaint_write },
1803     { .name = "C9", .cp = 15, .crn = 9,
1804       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW,
1805       .type = ARM_CP_CONST | ARM_CP_OVERRIDE, .resetvalue = 0 },
1806     REGINFO_SENTINEL
1807 };
1808
1809 static void xscale_cpar_write(CPUARMState *env, const ARMCPRegInfo *ri,
1810                               uint64_t value)
1811 {
1812     env->cp15.c15_cpar = value & 0x3fff;
1813 }
1814
1815 static const ARMCPRegInfo xscale_cp_reginfo[] = {
1816     { .name = "XSCALE_CPAR",
1817       .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0, .access = PL1_RW,
1818       .fieldoffset = offsetof(CPUARMState, cp15.c15_cpar), .resetvalue = 0,
1819       .writefn = xscale_cpar_write, },
1820     { .name = "XSCALE_AUXCR",
1821       .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1, .access = PL1_RW,
1822       .fieldoffset = offsetof(CPUARMState, cp15.c1_xscaleauxcr),
1823       .resetvalue = 0, },
1824     /* XScale specific cache-lockdown: since we have no cache we NOP these
1825      * and hope the guest does not really rely on cache behaviour.
1826      */
1827     { .name = "XSCALE_LOCK_ICACHE_LINE",
1828       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 0,
1829       .access = PL1_W, .type = ARM_CP_NOP },
1830     { .name = "XSCALE_UNLOCK_ICACHE",
1831       .cp = 15, .opc1 = 0, .crn = 9, .crm = 1, .opc2 = 1,
1832       .access = PL1_W, .type = ARM_CP_NOP },
1833     { .name = "XSCALE_DCACHE_LOCK",
1834       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 0,
1835       .access = PL1_RW, .type = ARM_CP_NOP },
1836     { .name = "XSCALE_UNLOCK_DCACHE",
1837       .cp = 15, .opc1 = 0, .crn = 9, .crm = 2, .opc2 = 1,
1838       .access = PL1_W, .type = ARM_CP_NOP },
1839     REGINFO_SENTINEL
1840 };
1841
1842 static const ARMCPRegInfo dummy_c15_cp_reginfo[] = {
1843     /* RAZ/WI the whole crn=15 space, when we don't have a more specific
1844      * implementation of this implementation-defined space.
1845      * Ideally this should eventually disappear in favour of actually
1846      * implementing the correct behaviour for all cores.
1847      */
1848     { .name = "C15_IMPDEF", .cp = 15, .crn = 15,
1849       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
1850       .access = PL1_RW,
1851       .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE | ARM_CP_OVERRIDE,
1852       .resetvalue = 0 },
1853     REGINFO_SENTINEL
1854 };
1855
1856 static const ARMCPRegInfo cache_dirty_status_cp_reginfo[] = {
1857     /* Cache status: RAZ because we have no cache so it's always clean */
1858     { .name = "CDSR", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 6,
1859       .access = PL1_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
1860       .resetvalue = 0 },
1861     REGINFO_SENTINEL
1862 };
1863
1864 static const ARMCPRegInfo cache_block_ops_cp_reginfo[] = {
1865     /* We never have a a block transfer operation in progress */
1866     { .name = "BXSR", .cp = 15, .crn = 7, .crm = 12, .opc1 = 0, .opc2 = 4,
1867       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
1868       .resetvalue = 0 },
1869     /* The cache ops themselves: these all NOP for QEMU */
1870     { .name = "IICR", .cp = 15, .crm = 5, .opc1 = 0,
1871       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
1872     { .name = "IDCR", .cp = 15, .crm = 6, .opc1 = 0,
1873       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
1874     { .name = "CDCR", .cp = 15, .crm = 12, .opc1 = 0,
1875       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
1876     { .name = "PIR", .cp = 15, .crm = 12, .opc1 = 1,
1877       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
1878     { .name = "PDR", .cp = 15, .crm = 12, .opc1 = 2,
1879       .access = PL0_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
1880     { .name = "CIDCR", .cp = 15, .crm = 14, .opc1 = 0,
1881       .access = PL1_W, .type = ARM_CP_NOP|ARM_CP_64BIT },
1882     REGINFO_SENTINEL
1883 };
1884
1885 static const ARMCPRegInfo cache_test_clean_cp_reginfo[] = {
1886     /* The cache test-and-clean instructions always return (1 << 30)
1887      * to indicate that there are no dirty cache lines.
1888      */
1889     { .name = "TC_DCACHE", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 3,
1890       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
1891       .resetvalue = (1 << 30) },
1892     { .name = "TCI_DCACHE", .cp = 15, .crn = 7, .crm = 14, .opc1 = 0, .opc2 = 3,
1893       .access = PL0_R, .type = ARM_CP_CONST | ARM_CP_NO_MIGRATE,
1894       .resetvalue = (1 << 30) },
1895     REGINFO_SENTINEL
1896 };
1897
1898 static const ARMCPRegInfo strongarm_cp_reginfo[] = {
1899     /* Ignore ReadBuffer accesses */
1900     { .name = "C9_READBUFFER", .cp = 15, .crn = 9,
1901       .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
1902       .access = PL1_RW, .resetvalue = 0,
1903       .type = ARM_CP_CONST | ARM_CP_OVERRIDE | ARM_CP_NO_MIGRATE },
1904     REGINFO_SENTINEL
1905 };
1906
1907 static uint64_t mpidr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1908 {
1909     CPUState *cs = CPU(arm_env_get_cpu(env));
1910     uint32_t mpidr = cs->cpu_index;
1911     /* We don't support setting cluster ID ([8..11]) (known as Aff1
1912      * in later ARM ARM versions), or any of the higher affinity level fields,
1913      * so these bits always RAZ.
1914      */
1915     if (arm_feature(env, ARM_FEATURE_V7MP)) {
1916         mpidr |= (1U << 31);
1917         /* Cores which are uniprocessor (non-coherent)
1918          * but still implement the MP extensions set
1919          * bit 30. (For instance, A9UP.) However we do
1920          * not currently model any of those cores.
1921          */
1922     }
1923     return mpidr;
1924 }
1925
1926 static const ARMCPRegInfo mpidr_cp_reginfo[] = {
1927     { .name = "MPIDR", .state = ARM_CP_STATE_BOTH,
1928       .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 5,
1929       .access = PL1_R, .readfn = mpidr_read, .type = ARM_CP_NO_MIGRATE },
1930     REGINFO_SENTINEL
1931 };
1932
1933 static const ARMCPRegInfo lpae_cp_reginfo[] = {
1934     /* NOP AMAIR0/1: the override is because these clash with the rather
1935      * broadly specified TLB_LOCKDOWN entry in the generic cp_reginfo.
1936      */
1937     { .name = "AMAIR0", .state = ARM_CP_STATE_BOTH,
1938       .opc0 = 3, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 0,
1939       .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
1940       .resetvalue = 0 },
1941     /* AMAIR1 is mapped to AMAIR_EL1[63:32] */
1942     { .name = "AMAIR1", .cp = 15, .crn = 10, .crm = 3, .opc1 = 0, .opc2 = 1,
1943       .access = PL1_RW, .type = ARM_CP_CONST | ARM_CP_OVERRIDE,
1944       .resetvalue = 0 },
1945     { .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
1946       .access = PL1_RW, .type = ARM_CP_64BIT, .resetvalue = 0,
1947       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.par_s),
1948                              offsetof(CPUARMState, cp15.par_ns)} },
1949     { .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
1950       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE,
1951       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr0_s),
1952                              offsetof(CPUARMState, cp15.ttbr0_ns) },
1953       .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
1954     { .name = "TTBR1", .cp = 15, .crm = 2, .opc1 = 1,
1955       .access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE,
1956       .bank_fieldoffsets = { offsetof(CPUARMState, cp15.ttbr1_s),
1957                              offsetof(CPUARMState, cp15.ttbr1_ns) },
1958       .writefn = vmsa_ttbr_write, .resetfn = arm_cp_reset_ignore },
1959     REGINFO_SENTINEL
1960 };
1961
1962 static uint64_t aa64_fpcr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1963 {
1964     return vfp_get_fpcr(env);
1965 }
1966
1967 static void aa64_fpcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1968                             uint64_t value)
1969 {
1970     vfp_set_fpcr(env, value);
1971 }
1972
1973 static uint64_t aa64_fpsr_read(CPUARMState *env, const ARMCPRegInfo *ri)
1974 {
1975     return vfp_get_fpsr(env);
1976 }
1977
1978 static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
1979                             uint64_t value)
1980 {
1981     vfp_set_fpsr(env, value);
1982 }
1983
1984 static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri)
1985 {
1986     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UMA)) {
1987         return CP_ACCESS_TRAP;
1988     }
1989     return CP_ACCESS_OK;
1990 }
1991
1992 static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
1993                             uint64_t value)
1994 {
1995     env->daif = value & PSTATE_DAIF;
1996 }
1997
1998 static CPAccessResult aa64_cacheop_access(CPUARMState *env,
1999                                           const ARMCPRegInfo *ri)
2000 {
2001     /* Cache invalidate/clean: NOP, but EL0 must UNDEF unless
2002      * SCTLR_EL1.UCI is set.
2003      */
2004     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCI)) {
2005         return CP_ACCESS_TRAP;
2006     }
2007     return CP_ACCESS_OK;
2008 }
2009
2010 /* See: D4.7.2 TLB maintenance requirements and the TLB maintenance instructions
2011  * Page D4-1736 (DDI0487A.b)
2012  */
2013
2014 static void tlbi_aa64_va_write(CPUARMState *env, const ARMCPRegInfo *ri,
2015                                uint64_t value)
2016 {
2017     /* Invalidate by VA (AArch64 version) */
2018     ARMCPU *cpu = arm_env_get_cpu(env);
2019     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2020
2021     tlb_flush_page(CPU(cpu), pageaddr);
2022 }
2023
2024 static void tlbi_aa64_vaa_write(CPUARMState *env, const ARMCPRegInfo *ri,
2025                                 uint64_t value)
2026 {
2027     /* Invalidate by VA, all ASIDs (AArch64 version) */
2028     ARMCPU *cpu = arm_env_get_cpu(env);
2029     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2030
2031     tlb_flush_page(CPU(cpu), pageaddr);
2032 }
2033
2034 static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
2035                                  uint64_t value)
2036 {
2037     /* Invalidate by ASID (AArch64 version) */
2038     ARMCPU *cpu = arm_env_get_cpu(env);
2039     int asid = extract64(value, 48, 16);
2040     tlb_flush(CPU(cpu), asid == 0);
2041 }
2042
2043 static void tlbi_aa64_va_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2044                                   uint64_t value)
2045 {
2046     CPUState *other_cs;
2047     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2048
2049     CPU_FOREACH(other_cs) {
2050         tlb_flush_page(other_cs, pageaddr);
2051     }
2052 }
2053
2054 static void tlbi_aa64_vaa_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2055                                   uint64_t value)
2056 {
2057     CPUState *other_cs;
2058     uint64_t pageaddr = sextract64(value << 12, 0, 56);
2059
2060     CPU_FOREACH(other_cs) {
2061         tlb_flush_page(other_cs, pageaddr);
2062     }
2063 }
2064
2065 static void tlbi_aa64_asid_is_write(CPUARMState *env, const ARMCPRegInfo *ri,
2066                                   uint64_t value)
2067 {
2068     CPUState *other_cs;
2069     int asid = extract64(value, 48, 16);
2070
2071     CPU_FOREACH(other_cs) {
2072         tlb_flush(other_cs, asid == 0);
2073     }
2074 }
2075
2076 static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri)
2077 {
2078     /* We don't implement EL2, so the only control on DC ZVA is the
2079      * bit in the SCTLR which can prohibit access for EL0.
2080      */
2081     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_DZE)) {
2082         return CP_ACCESS_TRAP;
2083     }
2084     return CP_ACCESS_OK;
2085 }
2086
2087 static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
2088 {
2089     ARMCPU *cpu = arm_env_get_cpu(env);
2090     int dzp_bit = 1 << 4;
2091
2092     /* DZP indicates whether DC ZVA access is allowed */
2093     if (aa64_zva_access(env, NULL) == CP_ACCESS_OK) {
2094         dzp_bit = 0;
2095     }
2096     return cpu->dcz_blocksize | dzp_bit;
2097 }
2098
2099 static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
2100 {
2101     if (!(env->pstate & PSTATE_SP)) {
2102         /* Access to SP_EL0 is undefined if it's being used as
2103          * the stack pointer.
2104          */
2105         return CP_ACCESS_TRAP_UNCATEGORIZED;
2106     }
2107     return CP_ACCESS_OK;
2108 }
2109
2110 static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
2111 {
2112     return env->pstate & PSTATE_SP;
2113 }
2114
2115 static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
2116 {
2117     update_spsel(env, val);
2118 }
2119
2120 static void sctlr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2121                         uint64_t value)
2122 {
2123     ARMCPU *cpu = arm_env_get_cpu(env);
2124
2125     if (raw_read(env, ri) == value) {
2126         /* Skip the TLB flush if nothing actually changed; Linux likes
2127          * to do a lot of pointless SCTLR writes.
2128          */
2129         return;
2130     }
2131
2132     raw_write(env, ri, value);
2133     /* ??? Lots of these bits are not implemented.  */
2134     /* This may enable/disable the MMU, so do a TLB flush.  */
2135     tlb_flush(CPU(cpu), 1);
2136 }
2137
2138 static const ARMCPRegInfo v8_cp_reginfo[] = {
2139     /* Minimal set of EL0-visible registers. This will need to be expanded
2140      * significantly for system emulation of AArch64 CPUs.
2141      */
2142     { .name = "NZCV", .state = ARM_CP_STATE_AA64,
2143       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
2144       .access = PL0_RW, .type = ARM_CP_NZCV },
2145     { .name = "DAIF", .state = ARM_CP_STATE_AA64,
2146       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
2147       .type = ARM_CP_NO_MIGRATE,
2148       .access = PL0_RW, .accessfn = aa64_daif_access,
2149       .fieldoffset = offsetof(CPUARMState, daif),
2150       .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
2151     { .name = "FPCR", .state = ARM_CP_STATE_AA64,
2152       .opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
2153       .access = PL0_RW, .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
2154     { .name = "FPSR", .state = ARM_CP_STATE_AA64,
2155       .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
2156       .access = PL0_RW, .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
2157     { .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
2158       .opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
2159       .access = PL0_R, .type = ARM_CP_NO_MIGRATE,
2160       .readfn = aa64_dczid_read },
2161     { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
2162       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
2163       .access = PL0_W, .type = ARM_CP_DC_ZVA,
2164 #ifndef CONFIG_USER_ONLY
2165       /* Avoid overhead of an access check that always passes in user-mode */
2166       .accessfn = aa64_zva_access,
2167 #endif
2168     },
2169     { .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
2170       .opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
2171       .access = PL1_R, .type = ARM_CP_CURRENTEL },
2172     /* Cache ops: all NOPs since we don't emulate caches */
2173     { .name = "IC_IALLUIS", .state = ARM_CP_STATE_AA64,
2174       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
2175       .access = PL1_W, .type = ARM_CP_NOP },
2176     { .name = "IC_IALLU", .state = ARM_CP_STATE_AA64,
2177       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
2178       .access = PL1_W, .type = ARM_CP_NOP },
2179     { .name = "IC_IVAU", .state = ARM_CP_STATE_AA64,
2180       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 5, .opc2 = 1,
2181       .access = PL0_W, .type = ARM_CP_NOP,
2182       .accessfn = aa64_cacheop_access },
2183     { .name = "DC_IVAC", .state = ARM_CP_STATE_AA64,
2184       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
2185       .access = PL1_W, .type = ARM_CP_NOP },
2186     { .name = "DC_ISW", .state = ARM_CP_STATE_AA64,
2187       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
2188       .access = PL1_W, .type = ARM_CP_NOP },
2189     { .name = "DC_CVAC", .state = ARM_CP_STATE_AA64,
2190       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 10, .opc2 = 1,
2191       .access = PL0_W, .type = ARM_CP_NOP,
2192       .accessfn = aa64_cacheop_access },
2193     { .name = "DC_CSW", .state = ARM_CP_STATE_AA64,
2194       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
2195       .access = PL1_W, .type = ARM_CP_NOP },
2196     { .name = "DC_CVAU", .state = ARM_CP_STATE_AA64,
2197       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 11, .opc2 = 1,
2198       .access = PL0_W, .type = ARM_CP_NOP,
2199       .accessfn = aa64_cacheop_access },
2200     { .name = "DC_CIVAC", .state = ARM_CP_STATE_AA64,
2201       .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 14, .opc2 = 1,
2202       .access = PL0_W, .type = ARM_CP_NOP,
2203       .accessfn = aa64_cacheop_access },
2204     { .name = "DC_CISW", .state = ARM_CP_STATE_AA64,
2205       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
2206       .access = PL1_W, .type = ARM_CP_NOP },
2207     /* TLBI operations */
2208     { .name = "TLBI_VMALLE1IS", .state = ARM_CP_STATE_AA64,
2209       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
2210       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2211       .writefn = tlbiall_is_write },
2212     { .name = "TLBI_VAE1IS", .state = ARM_CP_STATE_AA64,
2213       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
2214       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2215       .writefn = tlbi_aa64_va_is_write },
2216     { .name = "TLBI_ASIDE1IS", .state = ARM_CP_STATE_AA64,
2217       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
2218       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2219       .writefn = tlbi_aa64_asid_is_write },
2220     { .name = "TLBI_VAAE1IS", .state = ARM_CP_STATE_AA64,
2221       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
2222       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2223       .writefn = tlbi_aa64_vaa_is_write },
2224     { .name = "TLBI_VALE1IS", .state = ARM_CP_STATE_AA64,
2225       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
2226       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2227       .writefn = tlbi_aa64_va_is_write },
2228     { .name = "TLBI_VAALE1IS", .state = ARM_CP_STATE_AA64,
2229       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
2230       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2231       .writefn = tlbi_aa64_vaa_is_write },
2232     { .name = "TLBI_VMALLE1", .state = ARM_CP_STATE_AA64,
2233       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
2234       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2235       .writefn = tlbiall_write },
2236     { .name = "TLBI_VAE1", .state = ARM_CP_STATE_AA64,
2237       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
2238       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2239       .writefn = tlbi_aa64_va_write },
2240     { .name = "TLBI_ASIDE1", .state = ARM_CP_STATE_AA64,
2241       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
2242       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2243       .writefn = tlbi_aa64_asid_write },
2244     { .name = "TLBI_VAAE1", .state = ARM_CP_STATE_AA64,
2245       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
2246       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2247       .writefn = tlbi_aa64_vaa_write },
2248     { .name = "TLBI_VALE1", .state = ARM_CP_STATE_AA64,
2249       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
2250       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2251       .writefn = tlbi_aa64_va_write },
2252     { .name = "TLBI_VAALE1", .state = ARM_CP_STATE_AA64,
2253       .opc0 = 1, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
2254       .access = PL1_W, .type = ARM_CP_NO_MIGRATE,
2255       .writefn = tlbi_aa64_vaa_write },
2256 #ifndef CONFIG_USER_ONLY
2257     /* 64 bit address translation operations */
2258     { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
2259       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
2260       .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
2261     { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
2262       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
2263       .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
2264     { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
2265       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
2266       .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
2267     { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
2268       .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
2269       .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
2270 #endif
2271     /* TLB invalidate last level of translation table walk */
2272     { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
2273       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_is_write },
2274     { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
2275       .type = ARM_CP_NO_MIGRATE, .access = PL1_W,
2276       .writefn = tlbimvaa_is_write },
2277     { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
2278       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
2279     { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
2280       .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
2281     /* 32 bit cache operations */
2282     { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
2283       .type = ARM_CP_NOP, .access = PL1_W },
2284     { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
2285       .type = ARM_CP_NOP, .access = PL1_W },
2286     { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
2287       .type = ARM_CP_NOP, .access = PL1_W },
2288     { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
2289       .type = ARM_CP_NOP, .access = PL1_W },
2290     { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
2291       .type = ARM_CP_NOP, .access = PL1_W },
2292     { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
2293       .type = ARM_CP_NOP, .access = PL1_W },
2294     { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
2295       .type = ARM_CP_NOP, .access = PL1_W },
2296     { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
2297       .type = ARM_CP_NOP, .access = PL1_W },
2298     { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
2299       .type = ARM_CP_NOP, .access = PL1_W },
2300     { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
2301       .type = ARM_CP_NOP, .access = PL1_W },
2302     { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
2303       .type = ARM_CP_NOP, .access = PL1_W },
2304     { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
2305       .type = ARM_CP_NOP, .access = PL1_W },
2306     { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
2307       .type = ARM_CP_NOP, .access = PL1_W },
2308     /* MMU Domain access control / MPU write buffer control */
2309     { .name = "DACR", .cp = 15, .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
2310       .access = PL1_RW, .resetvalue = 0,
2311       .writefn = dacr_write, .raw_writefn = raw_write,
2312       .bank_fieldoffsets = { offsetoflow32(CPUARMState, cp15.dacr_s),
2313                              offsetoflow32(CPUARMState, cp15.dacr_ns) } },
2314     { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
2315       .type = ARM_CP_NO_MIGRATE,
2316       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
2317       .access = PL1_RW,
2318       .fieldoffset = offsetof(CPUARMState, elr_el[1]) },
2319     { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
2320       .type = ARM_CP_NO_MIGRATE,
2321       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
2322       .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[0]) },
2323     /* We rely on the access checks not allowing the guest to write to the
2324      * state field when SPSel indicates that it's being used as the stack
2325      * pointer.
2326      */
2327     { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
2328       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
2329       .access = PL1_RW, .accessfn = sp_el0_access,
2330       .type = ARM_CP_NO_MIGRATE,
2331       .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
2332     { .name = "SPSel", .state = ARM_CP_STATE_AA64,
2333       .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
2334       .type = ARM_CP_NO_MIGRATE,
2335       .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
2336     REGINFO_SENTINEL
2337 };
2338
2339 /* Used to describe the behaviour of EL2 regs when EL2 does not exist.  */
2340 static const ARMCPRegInfo v8_el3_no_el2_cp_reginfo[] = {
2341     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
2342       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
2343       .access = PL2_RW,
2344       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
2345     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
2346       .type = ARM_CP_NO_MIGRATE,
2347       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
2348       .access = PL2_RW,
2349       .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore },
2350     REGINFO_SENTINEL
2351 };
2352
2353 static void hcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
2354 {
2355     ARMCPU *cpu = arm_env_get_cpu(env);
2356     uint64_t valid_mask = HCR_MASK;
2357
2358     if (arm_feature(env, ARM_FEATURE_EL3)) {
2359         valid_mask &= ~HCR_HCD;
2360     } else {
2361         valid_mask &= ~HCR_TSC;
2362     }
2363
2364     /* Clear RES0 bits.  */
2365     value &= valid_mask;
2366
2367     /* These bits change the MMU setup:
2368      * HCR_VM enables stage 2 translation
2369      * HCR_PTW forbids certain page-table setups
2370      * HCR_DC Disables stage1 and enables stage2 translation
2371      */
2372     if ((raw_read(env, ri) ^ value) & (HCR_VM | HCR_PTW | HCR_DC)) {
2373         tlb_flush(CPU(cpu), 1);
2374     }
2375     raw_write(env, ri, value);
2376 }
2377
2378 static const ARMCPRegInfo v8_el2_cp_reginfo[] = {
2379     { .name = "HCR_EL2", .state = ARM_CP_STATE_AA64,
2380       .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 1, .opc2 = 0,
2381       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.hcr_el2),
2382       .writefn = hcr_write },
2383     { .name = "DACR32_EL2", .state = ARM_CP_STATE_AA64,
2384       .opc0 = 3, .opc1 = 4, .crn = 3, .crm = 0, .opc2 = 0,
2385       .access = PL2_RW, .resetvalue = 0,
2386       .writefn = dacr_write, .raw_writefn = raw_write,
2387       .fieldoffset = offsetof(CPUARMState, cp15.dacr32_el2) },
2388     { .name = "ELR_EL2", .state = ARM_CP_STATE_AA64,
2389       .type = ARM_CP_NO_MIGRATE,
2390       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 1,
2391       .access = PL2_RW,
2392       .fieldoffset = offsetof(CPUARMState, elr_el[2]) },
2393     { .name = "ESR_EL2", .state = ARM_CP_STATE_AA64,
2394       .type = ARM_CP_NO_MIGRATE,
2395       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 2, .opc2 = 0,
2396       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[2]) },
2397     { .name = "IFSR32_EL2", .state = ARM_CP_STATE_AA64,
2398       .opc0 = 3, .opc1 = 4, .crn = 5, .crm = 0, .opc2 = 1,
2399       .access = PL2_RW, .resetvalue = 0,
2400       .fieldoffset = offsetof(CPUARMState, cp15.ifsr32_el2) },
2401     { .name = "FAR_EL2", .state = ARM_CP_STATE_AA64,
2402       .opc0 = 3, .opc1 = 4, .crn = 6, .crm = 0, .opc2 = 0,
2403       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[2]) },
2404     { .name = "SPSR_EL2", .state = ARM_CP_STATE_AA64,
2405       .type = ARM_CP_NO_MIGRATE,
2406       .opc0 = 3, .opc1 = 4, .crn = 4, .crm = 0, .opc2 = 0,
2407       .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[6]) },
2408     { .name = "VBAR_EL2", .state = ARM_CP_STATE_AA64,
2409       .opc0 = 3, .opc1 = 4, .crn = 12, .crm = 0, .opc2 = 0,
2410       .access = PL2_RW, .writefn = vbar_write,
2411       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[2]),
2412       .resetvalue = 0 },
2413     REGINFO_SENTINEL
2414 };
2415
2416 static const ARMCPRegInfo el3_cp_reginfo[] = {
2417     { .name = "SCR_EL3", .state = ARM_CP_STATE_AA64,
2418       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 0,
2419       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.scr_el3),
2420       .resetvalue = 0, .writefn = scr_write },
2421     { .name = "SCR",  .type = ARM_CP_NO_MIGRATE,
2422       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 0,
2423       .access = PL3_RW, .fieldoffset = offsetoflow32(CPUARMState, cp15.scr_el3),
2424       .resetfn = arm_cp_reset_ignore, .writefn = scr_write },
2425     { .name = "SDER32_EL3", .state = ARM_CP_STATE_AA64,
2426       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 1, .opc2 = 1,
2427       .access = PL3_RW, .resetvalue = 0,
2428       .fieldoffset = offsetof(CPUARMState, cp15.sder) },
2429     { .name = "SDER",
2430       .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 1,
2431       .access = PL3_RW, .resetvalue = 0,
2432       .fieldoffset = offsetoflow32(CPUARMState, cp15.sder) },
2433       /* TODO: Implement NSACR trapping of secure EL1 accesses to EL3 */
2434     { .name = "NSACR", .cp = 15, .opc1 = 0, .crn = 1, .crm = 1, .opc2 = 2,
2435       .access = PL3_W | PL1_R, .resetvalue = 0,
2436       .fieldoffset = offsetof(CPUARMState, cp15.nsacr) },
2437     { .name = "MVBAR", .cp = 15, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 1,
2438       .access = PL3_RW, .writefn = vbar_write, .resetvalue = 0,
2439       .fieldoffset = offsetof(CPUARMState, cp15.mvbar) },
2440     { .name = "SCTLR_EL3", .state = ARM_CP_STATE_AA64,
2441       .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 0, .opc2 = 0,
2442       .access = PL3_RW, .raw_writefn = raw_write, .writefn = sctlr_write,
2443       .fieldoffset = offsetof(CPUARMState, cp15.sctlr_el[3]) },
2444     { .name = "TTBR0_EL3", .state = ARM_CP_STATE_AA64,
2445       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 0,
2446       .access = PL3_RW, .writefn = vmsa_ttbr_write, .resetvalue = 0,
2447       .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el[3]) },
2448     { .name = "TCR_EL3", .state = ARM_CP_STATE_AA64,
2449       .opc0 = 3, .opc1 = 6, .crn = 2, .crm = 0, .opc2 = 2,
2450       .access = PL3_RW, .writefn = vmsa_tcr_el1_write,
2451       .resetfn = vmsa_ttbcr_reset, .raw_writefn = raw_write,
2452       .fieldoffset = offsetof(CPUARMState, cp15.tcr_el[3]) },
2453     { .name = "ELR_EL3", .state = ARM_CP_STATE_AA64,
2454       .type = ARM_CP_NO_MIGRATE,
2455       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 1,
2456       .access = PL3_RW,
2457       .fieldoffset = offsetof(CPUARMState, elr_el[3]) },
2458     { .name = "ESR_EL3", .state = ARM_CP_STATE_AA64,
2459       .type = ARM_CP_NO_MIGRATE,
2460       .opc0 = 3, .opc1 = 6, .crn = 5, .crm = 2, .opc2 = 0,
2461       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.esr_el[3]) },
2462     { .name = "FAR_EL3", .state = ARM_CP_STATE_AA64,
2463       .opc0 = 3, .opc1 = 6, .crn = 6, .crm = 0, .opc2 = 0,
2464       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el[3]) },
2465     { .name = "SPSR_EL3", .state = ARM_CP_STATE_AA64,
2466       .type = ARM_CP_NO_MIGRATE,
2467       .opc0 = 3, .opc1 = 6, .crn = 4, .crm = 0, .opc2 = 0,
2468       .access = PL3_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[7]) },
2469     { .name = "VBAR_EL3", .state = ARM_CP_STATE_AA64,
2470       .opc0 = 3, .opc1 = 6, .crn = 12, .crm = 0, .opc2 = 0,
2471       .access = PL3_RW, .writefn = vbar_write,
2472       .fieldoffset = offsetof(CPUARMState, cp15.vbar_el[3]),
2473       .resetvalue = 0 },
2474     REGINFO_SENTINEL
2475 };
2476
2477 static CPAccessResult ctr_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
2478 {
2479     /* Only accessible in EL0 if SCTLR.UCT is set (and only in AArch64,
2480      * but the AArch32 CTR has its own reginfo struct)
2481      */
2482     if (arm_current_el(env) == 0 && !(env->cp15.sctlr_el[1] & SCTLR_UCT)) {
2483         return CP_ACCESS_TRAP;
2484     }
2485     return CP_ACCESS_OK;
2486 }
2487
2488 static const ARMCPRegInfo debug_cp_reginfo[] = {
2489     /* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
2490      * debug components. The AArch64 version of DBGDRAR is named MDRAR_EL1;
2491      * unlike DBGDRAR it is never accessible from EL0.
2492      * DBGDSAR is deprecated and must RAZ from v8 anyway, so it has no AArch64
2493      * accessor.
2494      */
2495     { .name = "DBGDRAR", .cp = 14, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 0,
2496       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2497     { .name = "MDRAR_EL1", .state = ARM_CP_STATE_AA64,
2498       .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
2499       .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2500     { .name = "DBGDSAR", .cp = 14, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
2501       .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
2502     /* Monitor debug system control register; the 32-bit alias is DBGDSCRext. */
2503     { .name = "MDSCR_EL1", .state = ARM_CP_STATE_BOTH,
2504       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
2505       .access = PL1_RW,
2506       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
2507       .resetvalue = 0 },
2508     /* MDCCSR_EL0, aka DBGDSCRint. This is a read-only mirror of MDSCR_EL1.
2509      * We don't implement the configurable EL0 access.
2510      */
2511     { .name = "MDCCSR_EL0", .state = ARM_CP_STATE_BOTH,
2512       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
2513       .type = ARM_CP_NO_MIGRATE,
2514       .access = PL1_R,
2515       .fieldoffset = offsetof(CPUARMState, cp15.mdscr_el1),
2516       .resetfn = arm_cp_reset_ignore },
2517     /* We define a dummy WI OSLAR_EL1, because Linux writes to it. */
2518     { .name = "OSLAR_EL1", .state = ARM_CP_STATE_BOTH,
2519       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
2520       .access = PL1_W, .type = ARM_CP_NOP },
2521     /* Dummy OSDLR_EL1: 32-bit Linux will read this */
2522     { .name = "OSDLR_EL1", .state = ARM_CP_STATE_BOTH,
2523       .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 1, .crm = 3, .opc2 = 4,
2524       .access = PL1_RW, .type = ARM_CP_NOP },
2525     /* Dummy DBGVCR: Linux wants to clear this on startup, but we don't
2526      * implement vector catch debug events yet.
2527      */
2528     { .name = "DBGVCR",
2529       .cp = 14, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
2530       .access = PL1_RW, .type = ARM_CP_NOP },
2531     REGINFO_SENTINEL
2532 };
2533
2534 static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
2535     /* 64 bit access versions of the (dummy) debug registers */
2536     { .name = "DBGDRAR", .cp = 14, .crm = 1, .opc1 = 0,
2537       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
2538     { .name = "DBGDSAR", .cp = 14, .crm = 2, .opc1 = 0,
2539       .access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
2540     REGINFO_SENTINEL
2541 };
2542
2543 void hw_watchpoint_update(ARMCPU *cpu, int n)
2544 {
2545     CPUARMState *env = &cpu->env;
2546     vaddr len = 0;
2547     vaddr wvr = env->cp15.dbgwvr[n];
2548     uint64_t wcr = env->cp15.dbgwcr[n];
2549     int mask;
2550     int flags = BP_CPU | BP_STOP_BEFORE_ACCESS;
2551
2552     if (env->cpu_watchpoint[n]) {
2553         cpu_watchpoint_remove_by_ref(CPU(cpu), env->cpu_watchpoint[n]);
2554         env->cpu_watchpoint[n] = NULL;
2555     }
2556
2557     if (!extract64(wcr, 0, 1)) {
2558         /* E bit clear : watchpoint disabled */
2559         return;
2560     }
2561
2562     switch (extract64(wcr, 3, 2)) {
2563     case 0:
2564         /* LSC 00 is reserved and must behave as if the wp is disabled */
2565         return;
2566     case 1:
2567         flags |= BP_MEM_READ;
2568         break;
2569     case 2:
2570         flags |= BP_MEM_WRITE;
2571         break;
2572     case 3:
2573         flags |= BP_MEM_ACCESS;
2574         break;
2575     }
2576
2577     /* Attempts to use both MASK and BAS fields simultaneously are
2578      * CONSTRAINED UNPREDICTABLE; we opt to ignore BAS in this case,
2579      * thus generating a watchpoint for every byte in the masked region.
2580      */
2581     mask = extract64(wcr, 24, 4);
2582     if (mask == 1 || mask == 2) {
2583         /* Reserved values of MASK; we must act as if the mask value was
2584          * some non-reserved value, or as if the watchpoint were disabled.
2585          * We choose the latter.
2586          */
2587         return;
2588     } else if (mask) {
2589         /* Watchpoint covers an aligned area up to 2GB in size */
2590         len = 1ULL << mask;
2591         /* If masked bits in WVR are not zero it's CONSTRAINED UNPREDICTABLE
2592          * whether the watchpoint fires when the unmasked bits match; we opt
2593          * to generate the exceptions.
2594          */
2595         wvr &= ~(len - 1);
2596     } else {
2597         /* Watchpoint covers bytes defined by the byte address select bits */
2598         int bas = extract64(wcr, 5, 8);
2599         int basstart;
2600
2601         if (bas == 0) {
2602             /* This must act as if the watchpoint is disabled */
2603             return;
2604         }
2605
2606         if (extract64(wvr, 2, 1)) {
2607             /* Deprecated case of an only 4-aligned address. BAS[7:4] are
2608              * ignored, and BAS[3:0] define which bytes to watch.
2609              */
2610             bas &= 0xf;
2611         }
2612         /* The BAS bits are supposed to be programmed to indicate a contiguous
2613          * range of bytes. Otherwise it is CONSTRAINED UNPREDICTABLE whether
2614          * we fire for each byte in the word/doubleword addressed by the WVR.
2615          * We choose to ignore any non-zero bits after the first range of 1s.
2616          */
2617         basstart = ctz32(bas);
2618         len = cto32(bas >> basstart);
2619         wvr += basstart;
2620     }
2621
2622     cpu_watchpoint_insert(CPU(cpu), wvr, len, flags,
2623                           &env->cpu_watchpoint[n]);
2624 }
2625
2626 void hw_watchpoint_update_all(ARMCPU *cpu)
2627 {
2628     int i;
2629     CPUARMState *env = &cpu->env;
2630
2631     /* Completely clear out existing QEMU watchpoints and our array, to
2632      * avoid possible stale entries following migration load.
2633      */
2634     cpu_watchpoint_remove_all(CPU(cpu), BP_CPU);
2635     memset(env->cpu_watchpoint, 0, sizeof(env->cpu_watchpoint));
2636
2637     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_watchpoint); i++) {
2638         hw_watchpoint_update(cpu, i);
2639     }
2640 }
2641
2642 static void dbgwvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2643                          uint64_t value)
2644 {
2645     ARMCPU *cpu = arm_env_get_cpu(env);
2646     int i = ri->crm;
2647
2648     /* Bits [63:49] are hardwired to the value of bit [48]; that is, the
2649      * register reads and behaves as if values written are sign extended.
2650      * Bits [1:0] are RES0.
2651      */
2652     value = sextract64(value, 0, 49) & ~3ULL;
2653
2654     raw_write(env, ri, value);
2655     hw_watchpoint_update(cpu, i);
2656 }
2657
2658 static void dbgwcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2659                          uint64_t value)
2660 {
2661     ARMCPU *cpu = arm_env_get_cpu(env);
2662     int i = ri->crm;
2663
2664     raw_write(env, ri, value);
2665     hw_watchpoint_update(cpu, i);
2666 }
2667
2668 void hw_breakpoint_update(ARMCPU *cpu, int n)
2669 {
2670     CPUARMState *env = &cpu->env;
2671     uint64_t bvr = env->cp15.dbgbvr[n];
2672     uint64_t bcr = env->cp15.dbgbcr[n];
2673     vaddr addr;
2674     int bt;
2675     int flags = BP_CPU;
2676
2677     if (env->cpu_breakpoint[n]) {
2678         cpu_breakpoint_remove_by_ref(CPU(cpu), env->cpu_breakpoint[n]);
2679         env->cpu_breakpoint[n] = NULL;
2680     }
2681
2682     if (!extract64(bcr, 0, 1)) {
2683         /* E bit clear : watchpoint disabled */
2684         return;
2685     }
2686
2687     bt = extract64(bcr, 20, 4);
2688
2689     switch (bt) {
2690     case 4: /* unlinked address mismatch (reserved if AArch64) */
2691     case 5: /* linked address mismatch (reserved if AArch64) */
2692         qemu_log_mask(LOG_UNIMP,
2693                       "arm: address mismatch breakpoint types not implemented");
2694         return;
2695     case 0: /* unlinked address match */
2696     case 1: /* linked address match */
2697     {
2698         /* Bits [63:49] are hardwired to the value of bit [48]; that is,
2699          * we behave as if the register was sign extended. Bits [1:0] are
2700          * RES0. The BAS field is used to allow setting breakpoints on 16
2701          * bit wide instructions; it is CONSTRAINED UNPREDICTABLE whether
2702          * a bp will fire if the addresses covered by the bp and the addresses
2703          * covered by the insn overlap but the insn doesn't start at the
2704          * start of the bp address range. We choose to require the insn and
2705          * the bp to have the same address. The constraints on writing to
2706          * BAS enforced in dbgbcr_write mean we have only four cases:
2707          *  0b0000  => no breakpoint
2708          *  0b0011  => breakpoint on addr
2709          *  0b1100  => breakpoint on addr + 2
2710          *  0b1111  => breakpoint on addr
2711          * See also figure D2-3 in the v8 ARM ARM (DDI0487A.c).
2712          */
2713         int bas = extract64(bcr, 5, 4);
2714         addr = sextract64(bvr, 0, 49) & ~3ULL;
2715         if (bas == 0) {
2716             return;
2717         }
2718         if (bas == 0xc) {
2719             addr += 2;
2720         }
2721         break;
2722     }
2723     case 2: /* unlinked context ID match */
2724     case 8: /* unlinked VMID match (reserved if no EL2) */
2725     case 10: /* unlinked context ID and VMID match (reserved if no EL2) */
2726         qemu_log_mask(LOG_UNIMP,
2727                       "arm: unlinked context breakpoint types not implemented");
2728         return;
2729     case 9: /* linked VMID match (reserved if no EL2) */
2730     case 11: /* linked context ID and VMID match (reserved if no EL2) */
2731     case 3: /* linked context ID match */
2732     default:
2733         /* We must generate no events for Linked context matches (unless
2734          * they are linked to by some other bp/wp, which is handled in
2735          * updates for the linking bp/wp). We choose to also generate no events
2736          * for reserved values.
2737          */
2738         return;
2739     }
2740
2741     cpu_breakpoint_insert(CPU(cpu), addr, flags, &env->cpu_breakpoint[n]);
2742 }
2743
2744 void hw_breakpoint_update_all(ARMCPU *cpu)
2745 {
2746     int i;
2747     CPUARMState *env = &cpu->env;
2748
2749     /* Completely clear out existing QEMU breakpoints and our array, to
2750      * avoid possible stale entries following migration load.
2751      */
2752     cpu_breakpoint_remove_all(CPU(cpu), BP_CPU);
2753     memset(env->cpu_breakpoint, 0, sizeof(env->cpu_breakpoint));
2754
2755     for (i = 0; i < ARRAY_SIZE(cpu->env.cpu_breakpoint); i++) {
2756         hw_breakpoint_update(cpu, i);
2757     }
2758 }
2759
2760 static void dbgbvr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2761                          uint64_t value)
2762 {
2763     ARMCPU *cpu = arm_env_get_cpu(env);
2764     int i = ri->crm;
2765
2766     raw_write(env, ri, value);
2767     hw_breakpoint_update(cpu, i);
2768 }
2769
2770 static void dbgbcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
2771                          uint64_t value)
2772 {
2773     ARMCPU *cpu = arm_env_get_cpu(env);
2774     int i = ri->crm;
2775
2776     /* BAS[3] is a read-only copy of BAS[2], and BAS[1] a read-only
2777      * copy of BAS[0].
2778      */
2779     value = deposit64(value, 6, 1, extract64(value, 5, 1));
2780     value = deposit64(value, 8, 1, extract64(value, 7, 1));
2781
2782     raw_write(env, ri, value);
2783     hw_breakpoint_update(cpu, i);
2784 }
2785
2786 static void define_debug_regs(ARMCPU *cpu)
2787 {
2788     /* Define v7 and v8 architectural debug registers.
2789      * These are just dummy implementations for now.
2790      */
2791     int i;
2792     int wrps, brps, ctx_cmps;
2793     ARMCPRegInfo dbgdidr = {
2794         .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
2795         .access = PL0_R, .type = ARM_CP_CONST, .resetvalue = cpu->dbgdidr,
2796     };
2797
2798     /* Note that all these register fields hold "number of Xs minus 1". */
2799     brps = extract32(cpu->dbgdidr, 24, 4);
2800     wrps = extract32(cpu->dbgdidr, 28, 4);
2801     ctx_cmps = extract32(cpu->dbgdidr, 20, 4);
2802
2803     assert(ctx_cmps <= brps);
2804
2805     /* The DBGDIDR and ID_AA64DFR0_EL1 define various properties
2806      * of the debug registers such as number of breakpoints;
2807      * check that if they both exist then they agree.
2808      */
2809     if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
2810         assert(extract32(cpu->id_aa64dfr0, 12, 4) == brps);
2811         assert(extract32(cpu->id_aa64dfr0, 20, 4) == wrps);
2812         assert(extract32(cpu->id_aa64dfr0, 28, 4) == ctx_cmps);
2813     }
2814
2815     define_one_arm_cp_reg(cpu, &dbgdidr);
2816     define_arm_cp_regs(cpu, debug_cp_reginfo);
2817
2818     if (arm_feature(&cpu->env, ARM_FEATURE_LPAE)) {
2819         define_arm_cp_regs(cpu, debug_lpae_cp_reginfo);
2820     }
2821
2822     for (i = 0; i < brps + 1; i++) {
2823         ARMCPRegInfo dbgregs[] = {
2824             { .name = "DBGBVR", .state = ARM_CP_STATE_BOTH,
2825               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 4,
2826               .access = PL1_RW,
2827               .fieldoffset = offsetof(CPUARMState, cp15.dbgbvr[i]),
2828               .writefn = dbgbvr_write, .raw_writefn = raw_write
2829             },
2830             { .name = "DBGBCR", .state = ARM_CP_STATE_BOTH,
2831               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 5,
2832               .access = PL1_RW,
2833               .fieldoffset = offsetof(CPUARMState, cp15.dbgbcr[i]),
2834               .writefn = dbgbcr_write, .raw_writefn = raw_write
2835             },
2836             REGINFO_SENTINEL
2837         };
2838         define_arm_cp_regs(cpu, dbgregs);
2839     }
2840
2841     for (i = 0; i < wrps + 1; i++) {
2842         ARMCPRegInfo dbgregs[] = {
2843             { .name = "DBGWVR", .state = ARM_CP_STATE_BOTH,
2844               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 6,
2845               .access = PL1_RW,
2846               .fieldoffset = offsetof(CPUARMState, cp15.dbgwvr[i]),
2847               .writefn = dbgwvr_write, .raw_writefn = raw_write
2848             },
2849             { .name = "DBGWCR", .state = ARM_CP_STATE_BOTH,
2850               .cp = 14, .opc0 = 2, .opc1 = 0, .crn = 0, .crm = i, .opc2 = 7,
2851               .access = PL1_RW,
2852               .fieldoffset = offsetof(CPUARMState, cp15.dbgwcr[i]),
2853               .writefn = dbgwcr_write, .raw_writefn = raw_write
2854             },
2855             REGINFO_SENTINEL
2856         };
2857         define_arm_cp_regs(cpu, dbgregs);
2858     }
2859 }
2860
2861 void register_cp_regs_for_features(ARMCPU *cpu)
2862 {
2863     /* Register all the coprocessor registers based on feature bits */
2864     CPUARMState *env = &cpu->env;
2865     if (arm_feature(env, ARM_FEATURE_M)) {
2866         /* M profile has no coprocessor registers */
2867         return;
2868     }
2869
2870     define_arm_cp_regs(cpu, cp_reginfo);
2871     if (!arm_feature(env, ARM_FEATURE_V8)) {
2872         /* Must go early as it is full of wildcards that may be
2873          * overridden by later definitions.
2874          */
2875         define_arm_cp_regs(cpu, not_v8_cp_reginfo);
2876     }
2877
2878     if (arm_feature(env, ARM_FEATURE_V6)) {
2879         /* The ID registers all have impdef reset values */
2880         ARMCPRegInfo v6_idregs[] = {
2881             { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
2882               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
2883               .access = PL1_R, .type = ARM_CP_CONST,
2884               .resetvalue = cpu->id_pfr0 },
2885             { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
2886               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
2887               .access = PL1_R, .type = ARM_CP_CONST,
2888               .resetvalue = cpu->id_pfr1 },
2889             { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
2890               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
2891               .access = PL1_R, .type = ARM_CP_CONST,
2892               .resetvalue = cpu->id_dfr0 },
2893             { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
2894               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
2895               .access = PL1_R, .type = ARM_CP_CONST,
2896               .resetvalue = cpu->id_afr0 },
2897             { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
2898               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
2899               .access = PL1_R, .type = ARM_CP_CONST,
2900               .resetvalue = cpu->id_mmfr0 },
2901             { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
2902               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
2903               .access = PL1_R, .type = ARM_CP_CONST,
2904               .resetvalue = cpu->id_mmfr1 },
2905             { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
2906               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
2907               .access = PL1_R, .type = ARM_CP_CONST,
2908               .resetvalue = cpu->id_mmfr2 },
2909             { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
2910               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
2911               .access = PL1_R, .type = ARM_CP_CONST,
2912               .resetvalue = cpu->id_mmfr3 },
2913             { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
2914               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
2915               .access = PL1_R, .type = ARM_CP_CONST,
2916               .resetvalue = cpu->id_isar0 },
2917             { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
2918               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
2919               .access = PL1_R, .type = ARM_CP_CONST,
2920               .resetvalue = cpu->id_isar1 },
2921             { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
2922               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
2923               .access = PL1_R, .type = ARM_CP_CONST,
2924               .resetvalue = cpu->id_isar2 },
2925             { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
2926               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
2927               .access = PL1_R, .type = ARM_CP_CONST,
2928               .resetvalue = cpu->id_isar3 },
2929             { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
2930               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
2931               .access = PL1_R, .type = ARM_CP_CONST,
2932               .resetvalue = cpu->id_isar4 },
2933             { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
2934               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
2935               .access = PL1_R, .type = ARM_CP_CONST,
2936               .resetvalue = cpu->id_isar5 },
2937             /* 6..7 are as yet unallocated and must RAZ */
2938             { .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
2939               .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
2940               .resetvalue = 0 },
2941             { .name = "ID_ISAR7", .cp = 15, .crn = 0, .crm = 2,
2942               .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
2943               .resetvalue = 0 },
2944             REGINFO_SENTINEL
2945         };
2946         define_arm_cp_regs(cpu, v6_idregs);
2947         define_arm_cp_regs(cpu, v6_cp_reginfo);
2948     } else {
2949         define_arm_cp_regs(cpu, not_v6_cp_reginfo);
2950     }
2951     if (arm_feature(env, ARM_FEATURE_V6K)) {
2952         define_arm_cp_regs(cpu, v6k_cp_reginfo);
2953     }
2954     if (arm_feature(env, ARM_FEATURE_V7MP)) {
2955         define_arm_cp_regs(cpu, v7mp_cp_reginfo);
2956     }
2957     if (arm_feature(env, ARM_FEATURE_V7)) {
2958         /* v7 performance monitor control register: same implementor
2959          * field as main ID register, and we implement only the cycle
2960          * count register.
2961          */
2962 #ifndef CONFIG_USER_ONLY
2963         ARMCPRegInfo pmcr = {
2964             .name = "PMCR", .cp = 15, .crn = 9, .crm = 12, .opc1 = 0, .opc2 = 0,
2965             .access = PL0_RW,
2966             .type = ARM_CP_IO | ARM_CP_NO_MIGRATE,
2967             .fieldoffset = offsetoflow32(CPUARMState, cp15.c9_pmcr),
2968             .accessfn = pmreg_access, .writefn = pmcr_write,
2969             .raw_writefn = raw_write,
2970         };
2971         ARMCPRegInfo pmcr64 = {
2972             .name = "PMCR_EL0", .state = ARM_CP_STATE_AA64,
2973             .opc0 = 3, .opc1 = 3, .crn = 9, .crm = 12, .opc2 = 0,
2974             .access = PL0_RW, .accessfn = pmreg_access,
2975             .type = ARM_CP_IO,
2976             .fieldoffset = offsetof(CPUARMState, cp15.c9_pmcr),
2977             .resetvalue = cpu->midr & 0xff000000,
2978             .writefn = pmcr_write, .raw_writefn = raw_write,
2979         };
2980         define_one_arm_cp_reg(cpu, &pmcr);
2981         define_one_arm_cp_reg(cpu, &pmcr64);
2982 #endif
2983         ARMCPRegInfo clidr = {
2984             .name = "CLIDR", .state = ARM_CP_STATE_BOTH,
2985             .opc0 = 3, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 1,
2986             .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->clidr
2987         };
2988         define_one_arm_cp_reg(cpu, &clidr);
2989         define_arm_cp_regs(cpu, v7_cp_reginfo);
2990         define_debug_regs(cpu);
2991     } else {
2992         define_arm_cp_regs(cpu, not_v7_cp_reginfo);
2993     }
2994     if (arm_feature(env, ARM_FEATURE_V8)) {
2995         /* AArch64 ID registers, which all have impdef reset values */
2996         ARMCPRegInfo v8_idregs[] = {
2997             { .name = "ID_AA64PFR0_EL1", .state = ARM_CP_STATE_AA64,
2998               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 0,
2999               .access = PL1_R, .type = ARM_CP_CONST,
3000               .resetvalue = cpu->id_aa64pfr0 },
3001             { .name = "ID_AA64PFR1_EL1", .state = ARM_CP_STATE_AA64,
3002               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 1,
3003               .access = PL1_R, .type = ARM_CP_CONST,
3004               .resetvalue = cpu->id_aa64pfr1},
3005             { .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
3006               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
3007               .access = PL1_R, .type = ARM_CP_CONST,
3008               /* We mask out the PMUVer field, because we don't currently
3009                * implement the PMU. Not advertising it prevents the guest
3010                * from trying to use it and getting UNDEFs on registers we
3011                * don't implement.
3012                */
3013               .resetvalue = cpu->id_aa64dfr0 & ~0xf00 },
3014             { .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
3015               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
3016               .access = PL1_R, .type = ARM_CP_CONST,
3017               .resetvalue = cpu->id_aa64dfr1 },
3018             { .name = "ID_AA64AFR0_EL1", .state = ARM_CP_STATE_AA64,
3019               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 4,
3020               .access = PL1_R, .type = ARM_CP_CONST,
3021               .resetvalue = cpu->id_aa64afr0 },
3022             { .name = "ID_AA64AFR1_EL1", .state = ARM_CP_STATE_AA64,
3023               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 5,
3024               .access = PL1_R, .type = ARM_CP_CONST,
3025               .resetvalue = cpu->id_aa64afr1 },
3026             { .name = "ID_AA64ISAR0_EL1", .state = ARM_CP_STATE_AA64,
3027               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 0,
3028               .access = PL1_R, .type = ARM_CP_CONST,
3029               .resetvalue = cpu->id_aa64isar0 },
3030             { .name = "ID_AA64ISAR1_EL1", .state = ARM_CP_STATE_AA64,
3031               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 6, .opc2 = 1,
3032               .access = PL1_R, .type = ARM_CP_CONST,
3033               .resetvalue = cpu->id_aa64isar1 },
3034             { .name = "ID_AA64MMFR0_EL1", .state = ARM_CP_STATE_AA64,
3035               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 0,
3036               .access = PL1_R, .type = ARM_CP_CONST,
3037               .resetvalue = cpu->id_aa64mmfr0 },
3038             { .name = "ID_AA64MMFR1_EL1", .state = ARM_CP_STATE_AA64,
3039               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
3040               .access = PL1_R, .type = ARM_CP_CONST,
3041               .resetvalue = cpu->id_aa64mmfr1 },
3042             { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
3043               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
3044               .access = PL1_R, .type = ARM_CP_CONST,
3045               .resetvalue = cpu->mvfr0 },
3046             { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
3047               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
3048               .access = PL1_R, .type = ARM_CP_CONST,
3049               .resetvalue = cpu->mvfr1 },
3050             { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
3051               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
3052               .access = PL1_R, .type = ARM_CP_CONST,
3053               .resetvalue = cpu->mvfr2 },
3054             REGINFO_SENTINEL
3055         };
3056         ARMCPRegInfo rvbar = {
3057             .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
3058             .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 2,
3059             .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
3060         };
3061         define_one_arm_cp_reg(cpu, &rvbar);
3062         define_arm_cp_regs(cpu, v8_idregs);
3063         define_arm_cp_regs(cpu, v8_cp_reginfo);
3064     }
3065     if (arm_feature(env, ARM_FEATURE_EL2)) {
3066         define_arm_cp_regs(cpu, v8_el2_cp_reginfo);
3067     } else {
3068         /* If EL2 is missing but higher ELs are enabled, we need to
3069          * register the no_el2 reginfos.
3070          */
3071         if (arm_feature(env, ARM_FEATURE_EL3)) {
3072             define_arm_cp_regs(cpu, v8_el3_no_el2_cp_reginfo);
3073         }
3074     }
3075     if (arm_feature(env, ARM_FEATURE_EL3)) {
3076         define_arm_cp_regs(cpu, el3_cp_reginfo);
3077     }
3078     if (arm_feature(env, ARM_FEATURE_MPU)) {
3079         /* These are the MPU registers prior to PMSAv6. Any new
3080          * PMSA core later than the ARM946 will require that we
3081          * implement the PMSAv6 or PMSAv7 registers, which are
3082          * completely different.
3083          */
3084         assert(!arm_feature(env, ARM_FEATURE_V6));
3085         define_arm_cp_regs(cpu, pmsav5_cp_reginfo);
3086     } else {
3087         define_arm_cp_regs(cpu, vmsa_cp_reginfo);
3088     }
3089     if (arm_feature(env, ARM_FEATURE_THUMB2EE)) {
3090         define_arm_cp_regs(cpu, t2ee_cp_reginfo);
3091     }
3092     if (arm_feature(env, ARM_FEATURE_GENERIC_TIMER)) {
3093         define_arm_cp_regs(cpu, generic_timer_cp_reginfo);
3094     }
3095     if (arm_feature(env, ARM_FEATURE_VAPA)) {
3096         define_arm_cp_regs(cpu, vapa_cp_reginfo);
3097     }
3098     if (arm_feature(env, ARM_FEATURE_CACHE_TEST_CLEAN)) {
3099         define_arm_cp_regs(cpu, cache_test_clean_cp_reginfo);
3100     }
3101     if (arm_feature(env, ARM_FEATURE_CACHE_DIRTY_REG)) {
3102         define_arm_cp_regs(cpu, cache_dirty_status_cp_reginfo);
3103     }
3104     if (arm_feature(env, ARM_FEATURE_CACHE_BLOCK_OPS)) {
3105         define_arm_cp_regs(cpu, cache_block_ops_cp_reginfo);
3106     }
3107     if (arm_feature(env, ARM_FEATURE_OMAPCP)) {
3108         define_arm_cp_regs(cpu, omap_cp_reginfo);
3109     }
3110     if (arm_feature(env, ARM_FEATURE_STRONGARM)) {
3111         define_arm_cp_regs(cpu, strongarm_cp_reginfo);
3112     }
3113     if (arm_feature(env, ARM_FEATURE_XSCALE)) {
3114         define_arm_cp_regs(cpu, xscale_cp_reginfo);
3115     }
3116     if (arm_feature(env, ARM_FEATURE_DUMMY_C15_REGS)) {
3117         define_arm_cp_regs(cpu, dummy_c15_cp_reginfo);
3118     }
3119     if (arm_feature(env, ARM_FEATURE_LPAE)) {
3120         define_arm_cp_regs(cpu, lpae_cp_reginfo);
3121     }
3122     /* Slightly awkwardly, the OMAP and StrongARM cores need all of
3123      * cp15 crn=0 to be writes-ignored, whereas for other cores they should
3124      * be read-only (ie write causes UNDEF exception).
3125      */
3126     {
3127         ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
3128             /* Pre-v8 MIDR space.
3129              * Note that the MIDR isn't a simple constant register because
3130              * of the TI925 behaviour where writes to another register can
3131              * cause the MIDR value to change.
3132              *
3133              * Unimplemented registers in the c15 0 0 0 space default to
3134              * MIDR. Define MIDR first as this entire space, then CTR, TCMTR
3135              * and friends override accordingly.
3136              */
3137             { .name = "MIDR",
3138               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = CP_ANY,
3139               .access = PL1_R, .resetvalue = cpu->midr,
3140               .writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
3141               .fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
3142               .type = ARM_CP_OVERRIDE },
3143             /* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
3144             { .name = "DUMMY",
3145               .cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
3146               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3147             { .name = "DUMMY",
3148               .cp = 15, .crn = 0, .crm = 4, .opc1 = 0, .opc2 = CP_ANY,
3149               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3150             { .name = "DUMMY",
3151               .cp = 15, .crn = 0, .crm = 5, .opc1 = 0, .opc2 = CP_ANY,
3152               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3153             { .name = "DUMMY",
3154               .cp = 15, .crn = 0, .crm = 6, .opc1 = 0, .opc2 = CP_ANY,
3155               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3156             { .name = "DUMMY",
3157               .cp = 15, .crn = 0, .crm = 7, .opc1 = 0, .opc2 = CP_ANY,
3158               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3159             REGINFO_SENTINEL
3160         };
3161         ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
3162             /* v8 MIDR -- the wildcard isn't necessary, and nor is the
3163              * variable-MIDR TI925 behaviour. Instead we have a single
3164              * (strictly speaking IMPDEF) alias of the MIDR, REVIDR.
3165              */
3166             { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
3167               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
3168               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
3169             { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
3170               .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
3171               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
3172             REGINFO_SENTINEL
3173         };
3174         ARMCPRegInfo id_cp_reginfo[] = {
3175             /* These are common to v8 and pre-v8 */
3176             { .name = "CTR",
3177               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
3178               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
3179             { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
3180               .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
3181               .access = PL0_R, .accessfn = ctr_el0_access,
3182               .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
3183             /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
3184             { .name = "TCMTR",
3185               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
3186               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3187             { .name = "TLBTR",
3188               .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
3189               .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
3190             REGINFO_SENTINEL
3191         };
3192         ARMCPRegInfo crn0_wi_reginfo = {
3193             .name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
3194             .opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
3195             .type = ARM_CP_NOP | ARM_CP_OVERRIDE
3196         };
3197         if (arm_feature(env, ARM_FEATURE_OMAPCP) ||
3198             arm_feature(env, ARM_FEATURE_STRONGARM)) {
3199             ARMCPRegInfo *r;
3200             /* Register the blanket "writes ignored" value first to cover the
3201              * whole space. Then update the specific ID registers to allow write
3202              * access, so that they ignore writes rather than causing them to
3203              * UNDEF.
3204              */
3205             define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
3206             for (r = id_pre_v8_midr_cp_reginfo;
3207                  r->type != ARM_CP_SENTINEL; r++) {
3208                 r->access = PL1_RW;
3209             }
3210             for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
3211                 r->access = PL1_RW;
3212             }
3213         }
3214         if (arm_feature(env, ARM_FEATURE_V8)) {
3215             define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
3216         } else {
3217             define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
3218         }
3219         define_arm_cp_regs(cpu, id_cp_reginfo);
3220     }
3221
3222     if (arm_feature(env, ARM_FEATURE_MPIDR)) {
3223         define_arm_cp_regs(cpu, mpidr_cp_reginfo);
3224     }
3225
3226     if (arm_feature(env, ARM_FEATURE_AUXCR)) {
3227         ARMCPRegInfo auxcr = {
3228             .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
3229             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
3230             .access = PL1_RW, .type = ARM_CP_CONST,
3231             .resetvalue = cpu->reset_auxcr
3232         };
3233         define_one_arm_cp_reg(cpu, &auxcr);
3234     }
3235
3236     if (arm_feature(env, ARM_FEATURE_CBAR)) {
3237         if (arm_feature(env, ARM_FEATURE_AARCH64)) {
3238             /* 32 bit view is [31:18] 0...0 [43:32]. */
3239             uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
3240                 | extract64(cpu->reset_cbar, 32, 12);
3241             ARMCPRegInfo cbar_reginfo[] = {
3242                 { .name = "CBAR",
3243                   .type = ARM_CP_CONST,
3244                   .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
3245                   .access = PL1_R, .resetvalue = cpu->reset_cbar },
3246                 { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
3247                   .type = ARM_CP_CONST,
3248                   .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
3249                   .access = PL1_R, .resetvalue = cbar32 },
3250                 REGINFO_SENTINEL
3251             };
3252             /* We don't implement a r/w 64 bit CBAR currently */
3253             assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
3254             define_arm_cp_regs(cpu, cbar_reginfo);
3255         } else {
3256             ARMCPRegInfo cbar = {
3257                 .name = "CBAR",
3258                 .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
3259                 .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
3260                 .fieldoffset = offsetof(CPUARMState,
3261                                         cp15.c15_config_base_address)
3262             };
3263             if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
3264                 cbar.access = PL1_R;
3265                 cbar.fieldoffset = 0;
3266                 cbar.type = ARM_CP_CONST;
3267             }
3268             define_one_arm_cp_reg(cpu, &cbar);
3269         }
3270     }
3271
3272     /* Generic registers whose values depend on the implementation */
3273     {
3274         ARMCPRegInfo sctlr = {
3275             .name = "SCTLR", .state = ARM_CP_STATE_BOTH,
3276             .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 0,
3277             .access = PL1_RW,
3278             .bank_fieldoffsets = { offsetof(CPUARMState, cp15.sctlr_s),
3279                                    offsetof(CPUARMState, cp15.sctlr_ns) },
3280             .writefn = sctlr_write, .resetvalue = cpu->reset_sctlr,
3281             .raw_writefn = raw_write,
3282         };
3283         if (arm_feature(env, ARM_FEATURE_XSCALE)) {
3284             /* Normally we would always end the TB on an SCTLR write, but Linux
3285              * arch/arm/mach-pxa/sleep.S expects two instructions following
3286              * an MMU enable to execute from cache.  Imitate this behaviour.
3287              */
3288             sctlr.type |= ARM_CP_SUPPRESS_TB_END;
3289         }
3290         define_one_arm_cp_reg(cpu, &sctlr);
3291     }
3292 }
3293
3294 ARMCPU *cpu_arm_init(const char *cpu_model)
3295 {
3296     return ARM_CPU(cpu_generic_init(TYPE_ARM_CPU, cpu_model));
3297 }
3298
3299 void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
3300 {
3301     CPUState *cs = CPU(cpu);
3302     CPUARMState *env = &cpu->env;
3303
3304     if (arm_feature(env, ARM_FEATURE_AARCH64)) {
3305         gdb_register_coprocessor(cs, aarch64_fpu_gdb_get_reg,
3306                                  aarch64_fpu_gdb_set_reg,
3307                                  34, "aarch64-fpu.xml", 0);
3308     } else if (arm_feature(env, ARM_FEATURE_NEON)) {
3309         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3310                                  51, "arm-neon.xml", 0);
3311     } else if (arm_feature(env, ARM_FEATURE_VFP3)) {
3312         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3313                                  35, "arm-vfp3.xml", 0);
3314     } else if (arm_feature(env, ARM_FEATURE_VFP)) {
3315         gdb_register_coprocessor(cs, vfp_gdb_get_reg, vfp_gdb_set_reg,
3316                                  19, "arm-vfp.xml", 0);
3317     }
3318 }
3319
3320 /* Sort alphabetically by type name, except for "any". */
3321 static gint arm_cpu_list_compare(gconstpointer a, gconstpointer b)
3322 {
3323     ObjectClass *class_a = (ObjectClass *)a;
3324     ObjectClass *class_b = (ObjectClass *)b;
3325     const char *name_a, *name_b;
3326
3327     name_a = object_class_get_name(class_a);
3328     name_b = object_class_get_name(class_b);
3329     if (strcmp(name_a, "any-" TYPE_ARM_CPU) == 0) {
3330         return 1;
3331     } else if (strcmp(name_b, "any-" TYPE_ARM_CPU) == 0) {
3332         return -1;
3333     } else {
3334         return strcmp(name_a, name_b);
3335     }
3336 }
3337
3338 static void arm_cpu_list_entry(gpointer data, gpointer user_data)
3339 {
3340     ObjectClass *oc = data;
3341     CPUListState *s = user_data;
3342     const char *typename;
3343     char *name;
3344
3345     typename = object_class_get_name(oc);
3346     name = g_strndup(typename, strlen(typename) - strlen("-" TYPE_ARM_CPU));
3347     (*s->cpu_fprintf)(s->file, "  %s\n",
3348                       name);
3349     g_free(name);
3350 }
3351
3352 void arm_cpu_list(FILE *f, fprintf_function cpu_fprintf)
3353 {
3354     CPUListState s = {
3355         .file = f,
3356         .cpu_fprintf = cpu_fprintf,
3357     };
3358     GSList *list;
3359
3360     list = object_class_get_list(TYPE_ARM_CPU, false);
3361     list = g_slist_sort(list, arm_cpu_list_compare);
3362     (*cpu_fprintf)(f, "Available CPUs:\n");
3363     g_slist_foreach(list, arm_cpu_list_entry, &s);
3364     g_slist_free(list);
3365 #ifdef CONFIG_KVM
3366     /* The 'host' CPU type is dynamically registered only if KVM is
3367      * enabled, so we have to special-case it here:
3368      */
3369     (*cpu_fprintf)(f, "  host (only available in KVM mode)\n");
3370 #endif
3371 }
3372
3373 static void arm_cpu_add_definition(gpointer data, gpointer user_data)
3374 {
3375     ObjectClass *oc = data;
3376     CpuDefinitionInfoList **cpu_list = user_data;
3377     CpuDefinitionInfoList *entry;
3378     CpuDefinitionInfo *info;
3379     const char *typename;
3380
3381     typename = object_class_get_name(oc);
3382     info = g_malloc0(sizeof(*info));
3383     info->name = g_strndup(typename,
3384                            strlen(typename) - strlen("-" TYPE_ARM_CPU));
3385
3386     entry = g_malloc0(sizeof(*entry));
3387     entry->value = info;
3388     entry->next = *cpu_list;
3389     *cpu_list = entry;
3390 }
3391
3392 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
3393 {
3394     CpuDefinitionInfoList *cpu_list = NULL;
3395     GSList *list;
3396
3397     list = object_class_get_list(TYPE_ARM_CPU, false);
3398     g_slist_foreach(list, arm_cpu_add_definition, &cpu_list);
3399     g_slist_free(list);
3400
3401     return cpu_list;
3402 }
3403
3404 static void add_cpreg_to_hashtable(ARMCPU *cpu, const ARMCPRegInfo *r,
3405                                    void *opaque, int state, int secstate,
3406                                    int crm, int opc1, int opc2)
3407 {
3408     /* Private utility function for define_one_arm_cp_reg_with_opaque():
3409      * add a single reginfo struct to the hash table.
3410      */
3411     uint32_t *key = g_new(uint32_t, 1);
3412     ARMCPRegInfo *r2 = g_memdup(r, sizeof(ARMCPRegInfo));
3413     int is64 = (r->type & ARM_CP_64BIT) ? 1 : 0;
3414     int ns = (secstate & ARM_CP_SECSTATE_NS) ? 1 : 0;
3415
3416     /* Reset the secure state to the specific incoming state.  This is
3417      * necessary as the register may have been defined with both states.
3418      */
3419     r2->secure = secstate;
3420
3421     if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
3422         /* Register is banked (using both entries in array).
3423          * Overwriting fieldoffset as the array is only used to define
3424          * banked registers but later only fieldoffset is used.
3425          */
3426         r2->fieldoffset = r->bank_fieldoffsets[ns];
3427     }
3428
3429     if (state == ARM_CP_STATE_AA32) {
3430         if (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1]) {
3431             /* If the register is banked then we don't need to migrate or
3432              * reset the 32-bit instance in certain cases:
3433              *
3434              * 1) If the register has both 32-bit and 64-bit instances then we
3435              *    can count on the 64-bit instance taking care of the
3436              *    non-secure bank.
3437              * 2) If ARMv8 is enabled then we can count on a 64-bit version
3438              *    taking care of the secure bank.  This requires that separate
3439              *    32 and 64-bit definitions are provided.
3440              */
3441             if ((r->state == ARM_CP_STATE_BOTH && ns) ||
3442                 (arm_feature(&cpu->env, ARM_FEATURE_V8) && !ns)) {
3443                 r2->type |= ARM_CP_NO_MIGRATE;
3444                 r2->resetfn = arm_cp_reset_ignore;
3445             }
3446         } else if ((secstate != r->secure) && !ns) {
3447             /* The register is not banked so we only want to allow migration of
3448              * the non-secure instance.
3449              */
3450             r2->type |= ARM_CP_NO_MIGRATE;
3451             r2->resetfn = arm_cp_reset_ignore;
3452         }
3453
3454         if (r->state == ARM_CP_STATE_BOTH) {
3455             /* We assume it is a cp15 register if the .cp field is left unset.
3456              */
3457             if (r2->cp == 0) {
3458                 r2->cp = 15;
3459             }
3460
3461 #ifdef HOST_WORDS_BIGENDIAN
3462             if (r2->fieldoffset) {
3463                 r2->fieldoffset += sizeof(uint32_t);
3464             }
3465 #endif
3466         }
3467     }
3468     if (state == ARM_CP_STATE_AA64) {
3469         /* To allow abbreviation of ARMCPRegInfo
3470          * definitions, we treat cp == 0 as equivalent to
3471          * the value for "standard guest-visible sysreg".
3472          * STATE_BOTH definitions are also always "standard
3473          * sysreg" in their AArch64 view (the .cp value may
3474          * be non-zero for the benefit of the AArch32 view).
3475          */
3476         if (r->cp == 0 || r->state == ARM_CP_STATE_BOTH) {
3477             r2->cp = CP_REG_ARM64_SYSREG_CP;
3478         }
3479         *key = ENCODE_AA64_CP_REG(r2->cp, r2->crn, crm,
3480                                   r2->opc0, opc1, opc2);
3481     } else {
3482         *key = ENCODE_CP_REG(r2->cp, is64, ns, r2->crn, crm, opc1, opc2);
3483     }
3484     if (opaque) {
3485         r2->opaque = opaque;
3486     }
3487     /* reginfo passed to helpers is correct for the actual access,
3488      * and is never ARM_CP_STATE_BOTH:
3489      */
3490     r2->state = state;
3491     /* Make sure reginfo passed to helpers for wildcarded regs
3492      * has the correct crm/opc1/opc2 for this reg, not CP_ANY:
3493      */
3494     r2->crm = crm;
3495     r2->opc1 = opc1;
3496     r2->opc2 = opc2;
3497     /* By convention, for wildcarded registers only the first
3498      * entry is used for migration; the others are marked as
3499      * NO_MIGRATE so we don't try to transfer the register
3500      * multiple times. Special registers (ie NOP/WFI) are
3501      * never migratable.
3502      */
3503     if ((r->type & ARM_CP_SPECIAL) ||
3504         ((r->crm == CP_ANY) && crm != 0) ||
3505         ((r->opc1 == CP_ANY) && opc1 != 0) ||
3506         ((r->opc2 == CP_ANY) && opc2 != 0)) {
3507         r2->type |= ARM_CP_NO_MIGRATE;
3508     }
3509
3510     /* Overriding of an existing definition must be explicitly
3511      * requested.
3512      */
3513     if (!(r->type & ARM_CP_OVERRIDE)) {
3514         ARMCPRegInfo *oldreg;
3515         oldreg = g_hash_table_lookup(cpu->cp_regs, key);
3516         if (oldreg && !(oldreg->type & ARM_CP_OVERRIDE)) {
3517             fprintf(stderr, "Register redefined: cp=%d %d bit "
3518                     "crn=%d crm=%d opc1=%d opc2=%d, "
3519                     "was %s, now %s\n", r2->cp, 32 + 32 * is64,
3520                     r2->crn, r2->crm, r2->opc1, r2->opc2,
3521                     oldreg->name, r2->name);
3522             g_assert_not_reached();
3523         }
3524     }
3525     g_hash_table_insert(cpu->cp_regs, key, r2);
3526 }
3527
3528
3529 void define_one_arm_cp_reg_with_opaque(ARMCPU *cpu,
3530                                        const ARMCPRegInfo *r, void *opaque)
3531 {
3532     /* Define implementations of coprocessor registers.
3533      * We store these in a hashtable because typically
3534      * there are less than 150 registers in a space which
3535      * is 16*16*16*8*8 = 262144 in size.
3536      * Wildcarding is supported for the crm, opc1 and opc2 fields.
3537      * If a register is defined twice then the second definition is
3538      * used, so this can be used to define some generic registers and
3539      * then override them with implementation specific variations.
3540      * At least one of the original and the second definition should
3541      * include ARM_CP_OVERRIDE in its type bits -- this is just a guard
3542      * against accidental use.
3543      *
3544      * The state field defines whether the register is to be
3545      * visible in the AArch32 or AArch64 execution state. If the
3546      * state is set to ARM_CP_STATE_BOTH then we synthesise a
3547      * reginfo structure for the AArch32 view, which sees the lower
3548      * 32 bits of the 64 bit register.
3549      *
3550      * Only registers visible in AArch64 may set r->opc0; opc0 cannot
3551      * be wildcarded. AArch64 registers are always considered to be 64
3552      * bits; the ARM_CP_64BIT* flag applies only to the AArch32 view of
3553      * the register, if any.
3554      */
3555     int crm, opc1, opc2, state;
3556     int crmmin = (r->crm == CP_ANY) ? 0 : r->crm;
3557     int crmmax = (r->crm == CP_ANY) ? 15 : r->crm;
3558     int opc1min = (r->opc1 == CP_ANY) ? 0 : r->opc1;
3559     int opc1max = (r->opc1 == CP_ANY) ? 7 : r->opc1;
3560     int opc2min = (r->opc2 == CP_ANY) ? 0 : r->opc2;
3561     int opc2max = (r->opc2 == CP_ANY) ? 7 : r->opc2;
3562     /* 64 bit registers have only CRm and Opc1 fields */
3563     assert(!((r->type & ARM_CP_64BIT) && (r->opc2 || r->crn)));
3564     /* op0 only exists in the AArch64 encodings */
3565     assert((r->state != ARM_CP_STATE_AA32) || (r->opc0 == 0));
3566     /* AArch64 regs are all 64 bit so ARM_CP_64BIT is meaningless */
3567     assert((r->state != ARM_CP_STATE_AA64) || !(r->type & ARM_CP_64BIT));
3568     /* The AArch64 pseudocode CheckSystemAccess() specifies that op1
3569      * encodes a minimum access level for the register. We roll this
3570      * runtime check into our general permission check code, so check
3571      * here that the reginfo's specified permissions are strict enough
3572      * to encompass the generic architectural permission check.
3573      */
3574     if (r->state != ARM_CP_STATE_AA32) {
3575         int mask = 0;
3576         switch (r->opc1) {
3577         case 0: case 1: case 2:
3578             /* min_EL EL1 */
3579             mask = PL1_RW;
3580             break;
3581         case 3:
3582             /* min_EL EL0 */
3583             mask = PL0_RW;
3584             break;
3585         case 4:
3586             /* min_EL EL2 */
3587             mask = PL2_RW;
3588             break;
3589         case 5:
3590             /* unallocated encoding, so not possible */
3591             assert(false);
3592             break;
3593         case 6:
3594             /* min_EL EL3 */
3595             mask = PL3_RW;
3596             break;
3597         case 7:
3598             /* min_EL EL1, secure mode only (we don't check the latter) */
3599             mask = PL1_RW;
3600             break;
3601         default:
3602             /* broken reginfo with out-of-range opc1 */
3603             assert(false);
3604             break;
3605         }
3606         /* assert our permissions are not too lax (stricter is fine) */
3607         assert((r->access & ~mask) == 0);
3608     }
3609
3610     /* Check that the register definition has enough info to handle
3611      * reads and writes if they are permitted.
3612      */
3613     if (!(r->type & (ARM_CP_SPECIAL|ARM_CP_CONST))) {
3614         if (r->access & PL3_R) {
3615             assert((r->fieldoffset ||
3616                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
3617                    r->readfn);
3618         }
3619         if (r->access & PL3_W) {
3620             assert((r->fieldoffset ||
3621                    (r->bank_fieldoffsets[0] && r->bank_fieldoffsets[1])) ||
3622                    r->writefn);
3623         }
3624     }
3625     /* Bad type field probably means missing sentinel at end of reg list */
3626     assert(cptype_valid(r->type));
3627     for (crm = crmmin; crm <= crmmax; crm++) {
3628         for (opc1 = opc1min; opc1 <= opc1max; opc1++) {
3629             for (opc2 = opc2min; opc2 <= opc2max; opc2++) {
3630                 for (state = ARM_CP_STATE_AA32;
3631                      state <= ARM_CP_STATE_AA64; state++) {
3632                     if (r->state != state && r->state != ARM_CP_STATE_BOTH) {
3633                         continue;
3634                     }
3635                     if (state == ARM_CP_STATE_AA32) {
3636                         /* Under AArch32 CP registers can be common
3637                          * (same for secure and non-secure world) or banked.
3638                          */
3639                         switch (r->secure) {
3640                         case ARM_CP_SECSTATE_S:
3641                         case ARM_CP_SECSTATE_NS:
3642                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3643                                                    r->secure, crm, opc1, opc2);
3644                             break;
3645                         default:
3646                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3647                                                    ARM_CP_SECSTATE_S,
3648                                                    crm, opc1, opc2);
3649                             add_cpreg_to_hashtable(cpu, r, opaque, state,
3650                                                    ARM_CP_SECSTATE_NS,
3651                                                    crm, opc1, opc2);
3652                             break;
3653                         }
3654                     } else {
3655                         /* AArch64 registers get mapped to non-secure instance
3656                          * of AArch32 */
3657                         add_cpreg_to_hashtable(cpu, r, opaque, state,
3658                                                ARM_CP_SECSTATE_NS,
3659                                                crm, opc1, opc2);
3660                     }
3661                 }
3662             }
3663         }
3664     }
3665 }
3666
3667 void define_arm_cp_regs_with_opaque(ARMCPU *cpu,
3668                                     const ARMCPRegInfo *regs, void *opaque)
3669 {
3670     /* Define a whole list of registers */
3671     const ARMCPRegInfo *r;
3672     for (r = regs; r->type != ARM_CP_SENTINEL; r++) {
3673         define_one_arm_cp_reg_with_opaque(cpu, r, opaque);
3674     }
3675 }
3676
3677 const ARMCPRegInfo *get_arm_cp_reginfo(GHashTable *cpregs, uint32_t encoded_cp)
3678 {
3679     return g_hash_table_lookup(cpregs, &encoded_cp);
3680 }
3681
3682 void arm_cp_write_ignore(CPUARMState *env, const ARMCPRegInfo *ri,
3683                          uint64_t value)
3684 {
3685     /* Helper coprocessor write function for write-ignore registers */
3686 }
3687
3688 uint64_t arm_cp_read_zero(CPUARMState *env, const ARMCPRegInfo *ri)
3689 {
3690     /* Helper coprocessor write function for read-as-zero registers */
3691     return 0;
3692 }
3693
3694 void arm_cp_reset_ignore(CPUARMState *env, const ARMCPRegInfo *opaque)
3695 {
3696     /* Helper coprocessor reset function for do-nothing-on-reset registers */
3697 }
3698
3699 static int bad_mode_switch(CPUARMState *env, int mode)
3700 {
3701     /* Return true if it is not valid for us to switch to
3702      * this CPU mode (ie all the UNPREDICTABLE cases in
3703      * the ARM ARM CPSRWriteByInstr pseudocode).
3704      */
3705     switch (mode) {
3706     case ARM_CPU_MODE_USR:
3707     case ARM_CPU_MODE_SYS:
3708     case ARM_CPU_MODE_SVC:
3709     case ARM_CPU_MODE_ABT:
3710     case ARM_CPU_MODE_UND:
3711     case ARM_CPU_MODE_IRQ:
3712     case ARM_CPU_MODE_FIQ:
3713         return 0;
3714     case ARM_CPU_MODE_MON:
3715         return !arm_is_secure(env);
3716     default:
3717         return 1;
3718     }
3719 }
3720
3721 uint32_t cpsr_read(CPUARMState *env)
3722 {
3723     int ZF;
3724     ZF = (env->ZF == 0);
3725     return env->uncached_cpsr | (env->NF & 0x80000000) | (ZF << 30) |
3726         (env->CF << 29) | ((env->VF & 0x80000000) >> 3) | (env->QF << 27)
3727         | (env->thumb << 5) | ((env->condexec_bits & 3) << 25)
3728         | ((env->condexec_bits & 0xfc) << 8)
3729         | (env->GE << 16) | (env->daif & CPSR_AIF);
3730 }
3731
3732 void cpsr_write(CPUARMState *env, uint32_t val, uint32_t mask)
3733 {
3734     uint32_t changed_daif;
3735
3736     if (mask & CPSR_NZCV) {
3737         env->ZF = (~val) & CPSR_Z;
3738         env->NF = val;
3739         env->CF = (val >> 29) & 1;
3740         env->VF = (val << 3) & 0x80000000;
3741     }
3742     if (mask & CPSR_Q)
3743         env->QF = ((val & CPSR_Q) != 0);
3744     if (mask & CPSR_T)
3745         env->thumb = ((val & CPSR_T) != 0);
3746     if (mask & CPSR_IT_0_1) {
3747         env->condexec_bits &= ~3;
3748         env->condexec_bits |= (val >> 25) & 3;
3749     }
3750     if (mask & CPSR_IT_2_7) {
3751         env->condexec_bits &= 3;
3752         env->condexec_bits |= (val >> 8) & 0xfc;
3753     }
3754     if (mask & CPSR_GE) {
3755         env->GE = (val >> 16) & 0xf;
3756     }
3757
3758     /* In a V7 implementation that includes the security extensions but does
3759      * not include Virtualization Extensions the SCR.FW and SCR.AW bits control
3760      * whether non-secure software is allowed to change the CPSR_F and CPSR_A
3761      * bits respectively.
3762      *
3763      * In a V8 implementation, it is permitted for privileged software to
3764      * change the CPSR A/F bits regardless of the SCR.AW/FW bits.
3765      */
3766     if (!arm_feature(env, ARM_FEATURE_V8) &&
3767         arm_feature(env, ARM_FEATURE_EL3) &&
3768         !arm_feature(env, ARM_FEATURE_EL2) &&
3769         !arm_is_secure(env)) {
3770
3771         changed_daif = (env->daif ^ val) & mask;
3772
3773         if (changed_daif & CPSR_A) {
3774             /* Check to see if we are allowed to change the masking of async
3775              * abort exceptions from a non-secure state.
3776              */
3777             if (!(env->cp15.scr_el3 & SCR_AW)) {
3778                 qemu_log_mask(LOG_GUEST_ERROR,
3779                               "Ignoring attempt to switch CPSR_A flag from "
3780                               "non-secure world with SCR.AW bit clear\n");
3781                 mask &= ~CPSR_A;
3782             }
3783         }
3784
3785         if (changed_daif & CPSR_F) {
3786             /* Check to see if we are allowed to change the masking of FIQ
3787              * exceptions from a non-secure state.
3788              */
3789             if (!(env->cp15.scr_el3 & SCR_FW)) {
3790                 qemu_log_mask(LOG_GUEST_ERROR,
3791                               "Ignoring attempt to switch CPSR_F flag from "
3792                               "non-secure world with SCR.FW bit clear\n");
3793                 mask &= ~CPSR_F;
3794             }
3795
3796             /* Check whether non-maskable FIQ (NMFI) support is enabled.
3797              * If this bit is set software is not allowed to mask
3798              * FIQs, but is allowed to set CPSR_F to 0.
3799              */
3800             if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_NMFI) &&
3801                 (val & CPSR_F)) {
3802                 qemu_log_mask(LOG_GUEST_ERROR,
3803                               "Ignoring attempt to enable CPSR_F flag "
3804                               "(non-maskable FIQ [NMFI] support enabled)\n");
3805                 mask &= ~CPSR_F;
3806             }
3807         }
3808     }
3809
3810     env->daif &= ~(CPSR_AIF & mask);
3811     env->daif |= val & CPSR_AIF & mask;
3812
3813     if ((env->uncached_cpsr ^ val) & mask & CPSR_M) {
3814         if (bad_mode_switch(env, val & CPSR_M)) {
3815             /* Attempt to switch to an invalid mode: this is UNPREDICTABLE.
3816              * We choose to ignore the attempt and leave the CPSR M field
3817              * untouched.
3818              */
3819             mask &= ~CPSR_M;
3820         } else {
3821             switch_mode(env, val & CPSR_M);
3822         }
3823     }
3824     mask &= ~CACHED_CPSR_BITS;
3825     env->uncached_cpsr = (env->uncached_cpsr & ~mask) | (val & mask);
3826 }
3827
3828 /* Sign/zero extend */
3829 uint32_t HELPER(sxtb16)(uint32_t x)
3830 {
3831     uint32_t res;
3832     res = (uint16_t)(int8_t)x;
3833     res |= (uint32_t)(int8_t)(x >> 16) << 16;
3834     return res;
3835 }
3836
3837 uint32_t HELPER(uxtb16)(uint32_t x)
3838 {
3839     uint32_t res;
3840     res = (uint16_t)(uint8_t)x;
3841     res |= (uint32_t)(uint8_t)(x >> 16) << 16;
3842     return res;
3843 }
3844
3845 uint32_t HELPER(clz)(uint32_t x)
3846 {
3847     return clz32(x);
3848 }
3849
3850 int32_t HELPER(sdiv)(int32_t num, int32_t den)
3851 {
3852     if (den == 0)
3853       return 0;
3854     if (num == INT_MIN && den == -1)
3855       return INT_MIN;
3856     return num / den;
3857 }
3858
3859 uint32_t HELPER(udiv)(uint32_t num, uint32_t den)
3860 {
3861     if (den == 0)
3862       return 0;
3863     return num / den;
3864 }
3865
3866 uint32_t HELPER(rbit)(uint32_t x)
3867 {
3868     x =  ((x & 0xff000000) >> 24)
3869        | ((x & 0x00ff0000) >> 8)
3870        | ((x & 0x0000ff00) << 8)
3871        | ((x & 0x000000ff) << 24);
3872     x =  ((x & 0xf0f0f0f0) >> 4)
3873        | ((x & 0x0f0f0f0f) << 4);
3874     x =  ((x & 0x88888888) >> 3)
3875        | ((x & 0x44444444) >> 1)
3876        | ((x & 0x22222222) << 1)
3877        | ((x & 0x11111111) << 3);
3878     return x;
3879 }
3880
3881 #if defined(CONFIG_USER_ONLY)
3882
3883 int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
3884                              int mmu_idx)
3885 {
3886     ARMCPU *cpu = ARM_CPU(cs);
3887     CPUARMState *env = &cpu->env;
3888
3889     env->exception.vaddress = address;
3890     if (rw == 2) {
3891         cs->exception_index = EXCP_PREFETCH_ABORT;
3892     } else {
3893         cs->exception_index = EXCP_DATA_ABORT;
3894     }
3895     return 1;
3896 }
3897
3898 /* These should probably raise undefined insn exceptions.  */
3899 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
3900 {
3901     ARMCPU *cpu = arm_env_get_cpu(env);
3902
3903     cpu_abort(CPU(cpu), "v7m_msr %d\n", reg);
3904 }
3905
3906 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
3907 {
3908     ARMCPU *cpu = arm_env_get_cpu(env);
3909
3910     cpu_abort(CPU(cpu), "v7m_mrs %d\n", reg);
3911     return 0;
3912 }
3913
3914 void switch_mode(CPUARMState *env, int mode)
3915 {
3916     ARMCPU *cpu = arm_env_get_cpu(env);
3917
3918     if (mode != ARM_CPU_MODE_USR) {
3919         cpu_abort(CPU(cpu), "Tried to switch out of user mode\n");
3920     }
3921 }
3922
3923 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
3924 {
3925     ARMCPU *cpu = arm_env_get_cpu(env);
3926
3927     cpu_abort(CPU(cpu), "banked r13 write\n");
3928 }
3929
3930 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
3931 {
3932     ARMCPU *cpu = arm_env_get_cpu(env);
3933
3934     cpu_abort(CPU(cpu), "banked r13 read\n");
3935     return 0;
3936 }
3937
3938 unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
3939 {
3940     return 1;
3941 }
3942
3943 #else
3944
3945 /* Map CPU modes onto saved register banks.  */
3946 int bank_number(int mode)
3947 {
3948     switch (mode) {
3949     case ARM_CPU_MODE_USR:
3950     case ARM_CPU_MODE_SYS:
3951         return 0;
3952     case ARM_CPU_MODE_SVC:
3953         return 1;
3954     case ARM_CPU_MODE_ABT:
3955         return 2;
3956     case ARM_CPU_MODE_UND:
3957         return 3;
3958     case ARM_CPU_MODE_IRQ:
3959         return 4;
3960     case ARM_CPU_MODE_FIQ:
3961         return 5;
3962     case ARM_CPU_MODE_HYP:
3963         return 6;
3964     case ARM_CPU_MODE_MON:
3965         return 7;
3966     }
3967     hw_error("bank number requested for bad CPSR mode value 0x%x\n", mode);
3968 }
3969
3970 void switch_mode(CPUARMState *env, int mode)
3971 {
3972     int old_mode;
3973     int i;
3974
3975     old_mode = env->uncached_cpsr & CPSR_M;
3976     if (mode == old_mode)
3977         return;
3978
3979     if (old_mode == ARM_CPU_MODE_FIQ) {
3980         memcpy (env->fiq_regs, env->regs + 8, 5 * sizeof(uint32_t));
3981         memcpy (env->regs + 8, env->usr_regs, 5 * sizeof(uint32_t));
3982     } else if (mode == ARM_CPU_MODE_FIQ) {
3983         memcpy (env->usr_regs, env->regs + 8, 5 * sizeof(uint32_t));
3984         memcpy (env->regs + 8, env->fiq_regs, 5 * sizeof(uint32_t));
3985     }
3986
3987     i = bank_number(old_mode);
3988     env->banked_r13[i] = env->regs[13];
3989     env->banked_r14[i] = env->regs[14];
3990     env->banked_spsr[i] = env->spsr;
3991
3992     i = bank_number(mode);
3993     env->regs[13] = env->banked_r13[i];
3994     env->regs[14] = env->banked_r14[i];
3995     env->spsr = env->banked_spsr[i];
3996 }
3997
3998 /* Physical Interrupt Target EL Lookup Table
3999  *
4000  * [ From ARM ARM section G1.13.4 (Table G1-15) ]
4001  *
4002  * The below multi-dimensional table is used for looking up the target
4003  * exception level given numerous condition criteria.  Specifically, the
4004  * target EL is based on SCR and HCR routing controls as well as the
4005  * currently executing EL and secure state.
4006  *
4007  *    Dimensions:
4008  *    target_el_table[2][2][2][2][2][4]
4009  *                    |  |  |  |  |  +--- Current EL
4010  *                    |  |  |  |  +------ Non-secure(0)/Secure(1)
4011  *                    |  |  |  +--------- HCR mask override
4012  *                    |  |  +------------ SCR exec state control
4013  *                    |  +--------------- SCR mask override
4014  *                    +------------------ 32-bit(0)/64-bit(1) EL3
4015  *
4016  *    The table values are as such:
4017  *    0-3 = EL0-EL3
4018  *     -1 = Cannot occur
4019  *
4020  * The ARM ARM target EL table includes entries indicating that an "exception
4021  * is not taken".  The two cases where this is applicable are:
4022  *    1) An exception is taken from EL3 but the SCR does not have the exception
4023  *    routed to EL3.
4024  *    2) An exception is taken from EL2 but the HCR does not have the exception
4025  *    routed to EL2.
4026  * In these two cases, the below table contain a target of EL1.  This value is
4027  * returned as it is expected that the consumer of the table data will check
4028  * for "target EL >= current EL" to ensure the exception is not taken.
4029  *
4030  *            SCR     HCR
4031  *         64  EA     AMO                 From
4032  *        BIT IRQ     IMO      Non-secure         Secure
4033  *        EL3 FIQ  RW FMO   EL0 EL1 EL2 EL3   EL0 EL1 EL2 EL3
4034  */
4035 const int8_t target_el_table[2][2][2][2][2][4] = {
4036     {{{{/* 0   0   0   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
4037        {/* 0   0   0   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},
4038       {{/* 0   0   1   0 */{ 1,  1,  2, -1 },{ 3, -1, -1,  3 },},
4039        {/* 0   0   1   1 */{ 2,  2,  2, -1 },{ 3, -1, -1,  3 },},},},
4040      {{{/* 0   1   0   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
4041        {/* 0   1   0   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},
4042       {{/* 0   1   1   0 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},
4043        {/* 0   1   1   1 */{ 3,  3,  3, -1 },{ 3, -1, -1,  3 },},},},},
4044     {{{{/* 1   0   0   0 */{ 1,  1,  2, -1 },{ 1,  1, -1,  1 },},
4045        {/* 1   0   0   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},
4046       {{/* 1   0   1   0 */{ 1,  1,  1, -1 },{ 1,  1, -1,  1 },},
4047        {/* 1   0   1   1 */{ 2,  2,  2, -1 },{ 1,  1, -1,  1 },},},},
4048      {{{/* 1   1   0   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
4049        {/* 1   1   0   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},
4050       {{/* 1   1   1   0 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},
4051        {/* 1   1   1   1 */{ 3,  3,  3, -1 },{ 3,  3, -1,  3 },},},},},
4052 };
4053
4054 /*
4055  * Determine the target EL for physical exceptions
4056  */
4057 static inline uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
4058                                         uint32_t cur_el, bool secure)
4059 {
4060     CPUARMState *env = cs->env_ptr;
4061     int rw = ((env->cp15.scr_el3 & SCR_RW) == SCR_RW);
4062     int scr;
4063     int hcr;
4064     int target_el;
4065     int is64 = arm_el_is_aa64(env, 3);
4066
4067     switch (excp_idx) {
4068     case EXCP_IRQ:
4069         scr = ((env->cp15.scr_el3 & SCR_IRQ) == SCR_IRQ);
4070         hcr = ((env->cp15.hcr_el2 & HCR_IMO) == HCR_IMO);
4071         break;
4072     case EXCP_FIQ:
4073         scr = ((env->cp15.scr_el3 & SCR_FIQ) == SCR_FIQ);
4074         hcr = ((env->cp15.hcr_el2 & HCR_FMO) == HCR_FMO);
4075         break;
4076     default:
4077         scr = ((env->cp15.scr_el3 & SCR_EA) == SCR_EA);
4078         hcr = ((env->cp15.hcr_el2 & HCR_AMO) == HCR_AMO);
4079         break;
4080     };
4081
4082     /* If HCR.TGE is set then HCR is treated as being 1 */
4083     hcr |= ((env->cp15.hcr_el2 & HCR_TGE) == HCR_TGE);
4084
4085     /* Perform a table-lookup for the target EL given the current state */
4086     target_el = target_el_table[is64][scr][rw][hcr][secure][cur_el];
4087
4088     assert(target_el > 0);
4089
4090     return target_el;
4091 }
4092
4093 /*
4094  * Determine the target EL for a given exception type.
4095  */
4096 unsigned int arm_excp_target_el(CPUState *cs, unsigned int excp_idx)
4097 {
4098     ARMCPU *cpu = ARM_CPU(cs);
4099     CPUARMState *env = &cpu->env;
4100     unsigned int cur_el = arm_current_el(env);
4101     unsigned int target_el;
4102     bool secure = arm_is_secure(env);
4103
4104     switch (excp_idx) {
4105     case EXCP_HVC:
4106     case EXCP_HYP_TRAP:
4107         target_el = 2;
4108         break;
4109     case EXCP_SMC:
4110         target_el = 3;
4111         break;
4112     case EXCP_FIQ:
4113     case EXCP_IRQ:
4114         target_el = arm_phys_excp_target_el(cs, excp_idx, cur_el, secure);
4115         break;
4116     case EXCP_VIRQ:
4117     case EXCP_VFIQ:
4118         target_el = 1;
4119         break;
4120     default:
4121         target_el = MAX(cur_el, 1);
4122         break;
4123     }
4124     return target_el;
4125 }
4126
4127 static void v7m_push(CPUARMState *env, uint32_t val)
4128 {
4129     CPUState *cs = CPU(arm_env_get_cpu(env));
4130
4131     env->regs[13] -= 4;
4132     stl_phys(cs->as, env->regs[13], val);
4133 }
4134
4135 static uint32_t v7m_pop(CPUARMState *env)
4136 {
4137     CPUState *cs = CPU(arm_env_get_cpu(env));
4138     uint32_t val;
4139
4140     val = ldl_phys(cs->as, env->regs[13]);
4141     env->regs[13] += 4;
4142     return val;
4143 }
4144
4145 /* Switch to V7M main or process stack pointer.  */
4146 static void switch_v7m_sp(CPUARMState *env, int process)
4147 {
4148     uint32_t tmp;
4149     if (env->v7m.current_sp != process) {
4150         tmp = env->v7m.other_sp;
4151         env->v7m.other_sp = env->regs[13];
4152         env->regs[13] = tmp;
4153         env->v7m.current_sp = process;
4154     }
4155 }
4156
4157 static void do_v7m_exception_exit(CPUARMState *env)
4158 {
4159     uint32_t type;
4160     uint32_t xpsr;
4161
4162     type = env->regs[15];
4163     if (env->v7m.exception != 0)
4164         armv7m_nvic_complete_irq(env->nvic, env->v7m.exception);
4165
4166     /* Switch to the target stack.  */
4167     switch_v7m_sp(env, (type & 4) != 0);
4168     /* Pop registers.  */
4169     env->regs[0] = v7m_pop(env);
4170     env->regs[1] = v7m_pop(env);
4171     env->regs[2] = v7m_pop(env);
4172     env->regs[3] = v7m_pop(env);
4173     env->regs[12] = v7m_pop(env);
4174     env->regs[14] = v7m_pop(env);
4175     env->regs[15] = v7m_pop(env);
4176     xpsr = v7m_pop(env);
4177     xpsr_write(env, xpsr, 0xfffffdff);
4178     /* Undo stack alignment.  */
4179     if (xpsr & 0x200)
4180         env->regs[13] |= 4;
4181     /* ??? The exception return type specifies Thread/Handler mode.  However
4182        this is also implied by the xPSR value. Not sure what to do
4183        if there is a mismatch.  */
4184     /* ??? Likewise for mismatches between the CONTROL register and the stack
4185        pointer.  */
4186 }
4187
4188 void arm_v7m_cpu_do_interrupt(CPUState *cs)
4189 {
4190     ARMCPU *cpu = ARM_CPU(cs);
4191     CPUARMState *env = &cpu->env;
4192     uint32_t xpsr = xpsr_read(env);
4193     uint32_t lr;
4194     uint32_t addr;
4195
4196     arm_log_exception(cs->exception_index);
4197
4198     lr = 0xfffffff1;
4199     if (env->v7m.current_sp)
4200         lr |= 4;
4201     if (env->v7m.exception == 0)
4202         lr |= 8;
4203
4204     /* For exceptions we just mark as pending on the NVIC, and let that
4205        handle it.  */
4206     /* TODO: Need to escalate if the current priority is higher than the
4207        one we're raising.  */
4208     switch (cs->exception_index) {
4209     case EXCP_UDEF:
4210         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE);
4211         return;
4212     case EXCP_SWI:
4213         /* The PC already points to the next instruction.  */
4214         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SVC);
4215         return;
4216     case EXCP_PREFETCH_ABORT:
4217     case EXCP_DATA_ABORT:
4218         /* TODO: if we implemented the MPU registers, this is where we
4219          * should set the MMFAR, etc from exception.fsr and exception.vaddress.
4220          */
4221         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_MEM);
4222         return;
4223     case EXCP_BKPT:
4224         if (semihosting_enabled) {
4225             int nr;
4226             nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
4227             if (nr == 0xab) {
4228                 env->regs[15] += 2;
4229                 env->regs[0] = do_arm_semihosting(env);
4230                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4231                 return;
4232             }
4233         }
4234         armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_DEBUG);
4235         return;
4236     case EXCP_IRQ:
4237         env->v7m.exception = armv7m_nvic_acknowledge_irq(env->nvic);
4238         break;
4239     case EXCP_EXCEPTION_EXIT:
4240         do_v7m_exception_exit(env);
4241         return;
4242     default:
4243         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
4244         return; /* Never happens.  Keep compiler happy.  */
4245     }
4246
4247     /* Align stack pointer.  */
4248     /* ??? Should only do this if Configuration Control Register
4249        STACKALIGN bit is set.  */
4250     if (env->regs[13] & 4) {
4251         env->regs[13] -= 4;
4252         xpsr |= 0x200;
4253     }
4254     /* Switch to the handler mode.  */
4255     v7m_push(env, xpsr);
4256     v7m_push(env, env->regs[15]);
4257     v7m_push(env, env->regs[14]);
4258     v7m_push(env, env->regs[12]);
4259     v7m_push(env, env->regs[3]);
4260     v7m_push(env, env->regs[2]);
4261     v7m_push(env, env->regs[1]);
4262     v7m_push(env, env->regs[0]);
4263     switch_v7m_sp(env, 0);
4264     /* Clear IT bits */
4265     env->condexec_bits = 0;
4266     env->regs[14] = lr;
4267     addr = ldl_phys(cs->as, env->v7m.vecbase + env->v7m.exception * 4);
4268     env->regs[15] = addr & 0xfffffffe;
4269     env->thumb = addr & 1;
4270 }
4271
4272 /* Handle a CPU exception.  */
4273 void arm_cpu_do_interrupt(CPUState *cs)
4274 {
4275     ARMCPU *cpu = ARM_CPU(cs);
4276     CPUARMState *env = &cpu->env;
4277     uint32_t addr;
4278     uint32_t mask;
4279     int new_mode;
4280     uint32_t offset;
4281     uint32_t moe;
4282
4283     assert(!IS_M(env));
4284
4285     arm_log_exception(cs->exception_index);
4286
4287     if (arm_is_psci_call(cpu, cs->exception_index)) {
4288         arm_handle_psci_call(cpu);
4289         qemu_log_mask(CPU_LOG_INT, "...handled as PSCI call\n");
4290         return;
4291     }
4292
4293     /* If this is a debug exception we must update the DBGDSCR.MOE bits */
4294     switch (env->exception.syndrome >> ARM_EL_EC_SHIFT) {
4295     case EC_BREAKPOINT:
4296     case EC_BREAKPOINT_SAME_EL:
4297         moe = 1;
4298         break;
4299     case EC_WATCHPOINT:
4300     case EC_WATCHPOINT_SAME_EL:
4301         moe = 10;
4302         break;
4303     case EC_AA32_BKPT:
4304         moe = 3;
4305         break;
4306     case EC_VECTORCATCH:
4307         moe = 5;
4308         break;
4309     default:
4310         moe = 0;
4311         break;
4312     }
4313
4314     if (moe) {
4315         env->cp15.mdscr_el1 = deposit64(env->cp15.mdscr_el1, 2, 4, moe);
4316     }
4317
4318     /* TODO: Vectored interrupt controller.  */
4319     switch (cs->exception_index) {
4320     case EXCP_UDEF:
4321         new_mode = ARM_CPU_MODE_UND;
4322         addr = 0x04;
4323         mask = CPSR_I;
4324         if (env->thumb)
4325             offset = 2;
4326         else
4327             offset = 4;
4328         break;
4329     case EXCP_SWI:
4330         if (semihosting_enabled) {
4331             /* Check for semihosting interrupt.  */
4332             if (env->thumb) {
4333                 mask = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
4334                     & 0xff;
4335             } else {
4336                 mask = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
4337                     & 0xffffff;
4338             }
4339             /* Only intercept calls from privileged modes, to provide some
4340                semblance of security.  */
4341             if (((mask == 0x123456 && !env->thumb)
4342                     || (mask == 0xab && env->thumb))
4343                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
4344                 env->regs[0] = do_arm_semihosting(env);
4345                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4346                 return;
4347             }
4348         }
4349         new_mode = ARM_CPU_MODE_SVC;
4350         addr = 0x08;
4351         mask = CPSR_I;
4352         /* The PC already points to the next instruction.  */
4353         offset = 0;
4354         break;
4355     case EXCP_BKPT:
4356         /* See if this is a semihosting syscall.  */
4357         if (env->thumb && semihosting_enabled) {
4358             mask = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
4359             if (mask == 0xab
4360                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
4361                 env->regs[15] += 2;
4362                 env->regs[0] = do_arm_semihosting(env);
4363                 qemu_log_mask(CPU_LOG_INT, "...handled as semihosting call\n");
4364                 return;
4365             }
4366         }
4367         env->exception.fsr = 2;
4368         /* Fall through to prefetch abort.  */
4369     case EXCP_PREFETCH_ABORT:
4370         A32_BANKED_CURRENT_REG_SET(env, ifsr, env->exception.fsr);
4371         A32_BANKED_CURRENT_REG_SET(env, ifar, env->exception.vaddress);
4372         qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
4373                       env->exception.fsr, (uint32_t)env->exception.vaddress);
4374         new_mode = ARM_CPU_MODE_ABT;
4375         addr = 0x0c;
4376         mask = CPSR_A | CPSR_I;
4377         offset = 4;
4378         break;
4379     case EXCP_DATA_ABORT:
4380         A32_BANKED_CURRENT_REG_SET(env, dfsr, env->exception.fsr);
4381         A32_BANKED_CURRENT_REG_SET(env, dfar, env->exception.vaddress);
4382         qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
4383                       env->exception.fsr,
4384                       (uint32_t)env->exception.vaddress);
4385         new_mode = ARM_CPU_MODE_ABT;
4386         addr = 0x10;
4387         mask = CPSR_A | CPSR_I;
4388         offset = 8;
4389         break;
4390     case EXCP_IRQ:
4391         new_mode = ARM_CPU_MODE_IRQ;
4392         addr = 0x18;
4393         /* Disable IRQ and imprecise data aborts.  */
4394         mask = CPSR_A | CPSR_I;
4395         offset = 4;
4396         if (env->cp15.scr_el3 & SCR_IRQ) {
4397             /* IRQ routed to monitor mode */
4398             new_mode = ARM_CPU_MODE_MON;
4399             mask |= CPSR_F;
4400         }
4401         break;
4402     case EXCP_FIQ:
4403         new_mode = ARM_CPU_MODE_FIQ;
4404         addr = 0x1c;
4405         /* Disable FIQ, IRQ and imprecise data aborts.  */
4406         mask = CPSR_A | CPSR_I | CPSR_F;
4407         if (env->cp15.scr_el3 & SCR_FIQ) {
4408             /* FIQ routed to monitor mode */
4409             new_mode = ARM_CPU_MODE_MON;
4410         }
4411         offset = 4;
4412         break;
4413     case EXCP_SMC:
4414         new_mode = ARM_CPU_MODE_MON;
4415         addr = 0x08;
4416         mask = CPSR_A | CPSR_I | CPSR_F;
4417         offset = 0;
4418         break;
4419     default:
4420         cpu_abort(cs, "Unhandled exception 0x%x\n", cs->exception_index);
4421         return; /* Never happens.  Keep compiler happy.  */
4422     }
4423
4424     if (new_mode == ARM_CPU_MODE_MON) {
4425         addr += env->cp15.mvbar;
4426     } else if (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_V) {
4427         /* High vectors. When enabled, base address cannot be remapped. */
4428         addr += 0xffff0000;
4429     } else {
4430         /* ARM v7 architectures provide a vector base address register to remap
4431          * the interrupt vector table.
4432          * This register is only followed in non-monitor mode, and is banked.
4433          * Note: only bits 31:5 are valid.
4434          */
4435         addr += A32_BANKED_CURRENT_REG_GET(env, vbar);
4436     }
4437
4438     if ((env->uncached_cpsr & CPSR_M) == ARM_CPU_MODE_MON) {
4439         env->cp15.scr_el3 &= ~SCR_NS;
4440     }
4441
4442     switch_mode (env, new_mode);
4443     /* For exceptions taken to AArch32 we must clear the SS bit in both
4444      * PSTATE and in the old-state value we save to SPSR_<mode>, so zero it now.
4445      */
4446     env->uncached_cpsr &= ~PSTATE_SS;
4447     env->spsr = cpsr_read(env);
4448     /* Clear IT bits.  */
4449     env->condexec_bits = 0;
4450     /* Switch to the new mode, and to the correct instruction set.  */
4451     env->uncached_cpsr = (env->uncached_cpsr & ~CPSR_M) | new_mode;
4452     env->daif |= mask;
4453     /* this is a lie, as the was no c1_sys on V4T/V5, but who cares
4454      * and we should just guard the thumb mode on V4 */
4455     if (arm_feature(env, ARM_FEATURE_V4T)) {
4456         env->thumb = (A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_TE) != 0;
4457     }
4458     env->regs[14] = env->regs[15] + offset;
4459     env->regs[15] = addr;
4460     cs->interrupt_request |= CPU_INTERRUPT_EXITTB;
4461 }
4462
4463 /* Check section/page access permissions.
4464    Returns the page protection flags, or zero if the access is not
4465    permitted.  */
4466 static inline int check_ap(CPUARMState *env, int ap, int domain_prot,
4467                            int access_type, int is_user)
4468 {
4469   int prot_ro;
4470
4471   if (domain_prot == 3) {
4472     return PAGE_READ | PAGE_WRITE;
4473   }
4474
4475   if (access_type == 1)
4476       prot_ro = 0;
4477   else
4478       prot_ro = PAGE_READ;
4479
4480   switch (ap) {
4481   case 0:
4482       if (arm_feature(env, ARM_FEATURE_V7)) {
4483           return 0;
4484       }
4485       if (access_type == 1)
4486           return 0;
4487       switch (A32_BANKED_CURRENT_REG_GET(env, sctlr) & (SCTLR_S | SCTLR_R)) {
4488       case SCTLR_S:
4489           return is_user ? 0 : PAGE_READ;
4490       case SCTLR_R:
4491           return PAGE_READ;
4492       default:
4493           return 0;
4494       }
4495   case 1:
4496       return is_user ? 0 : PAGE_READ | PAGE_WRITE;
4497   case 2:
4498       if (is_user)
4499           return prot_ro;
4500       else
4501           return PAGE_READ | PAGE_WRITE;
4502   case 3:
4503       return PAGE_READ | PAGE_WRITE;
4504   case 4: /* Reserved.  */
4505       return 0;
4506   case 5:
4507       return is_user ? 0 : prot_ro;
4508   case 6:
4509       return prot_ro;
4510   case 7:
4511       if (!arm_feature (env, ARM_FEATURE_V6K))
4512           return 0;
4513       return prot_ro;
4514   default:
4515       abort();
4516   }
4517 }
4518
4519 static bool get_level1_table_address(CPUARMState *env, uint32_t *table,
4520                                          uint32_t address)
4521 {
4522     /* Get the TCR bank based on our security state */
4523     TCR *tcr = &env->cp15.tcr_el[arm_is_secure(env) ? 3 : 1];
4524
4525     /* We only get here if EL1 is running in AArch32. If EL3 is running in
4526      * AArch32 there is a secure and non-secure instance of the translation
4527      * table registers.
4528      */
4529     if (address & tcr->mask) {
4530         if (tcr->raw_tcr & TTBCR_PD1) {
4531             /* Translation table walk disabled for TTBR1 */
4532             return false;
4533         }
4534         *table = A32_BANKED_CURRENT_REG_GET(env, ttbr1) & 0xffffc000;
4535     } else {
4536         if (tcr->raw_tcr & TTBCR_PD0) {
4537             /* Translation table walk disabled for TTBR0 */
4538             return false;
4539         }
4540         *table = A32_BANKED_CURRENT_REG_GET(env, ttbr0) & tcr->base_mask;
4541     }
4542     *table |= (address >> 18) & 0x3ffc;
4543     return true;
4544 }
4545
4546 static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
4547                             int is_user, hwaddr *phys_ptr,
4548                             int *prot, target_ulong *page_size)
4549 {
4550     CPUState *cs = CPU(arm_env_get_cpu(env));
4551     int code;
4552     uint32_t table;
4553     uint32_t desc;
4554     int type;
4555     int ap;
4556     int domain = 0;
4557     int domain_prot;
4558     hwaddr phys_addr;
4559
4560     /* Pagetable walk.  */
4561     /* Lookup l1 descriptor.  */
4562     if (!get_level1_table_address(env, &table, address)) {
4563         /* Section translation fault if page walk is disabled by PD0 or PD1 */
4564         code = 5;
4565         goto do_fault;
4566     }
4567     desc = ldl_phys(cs->as, table);
4568     type = (desc & 3);
4569     domain = (desc >> 5) & 0x0f;
4570     domain_prot = (A32_BANKED_CURRENT_REG_GET(env, dacr) >> (domain * 2)) & 3;
4571     if (type == 0) {
4572         /* Section translation fault.  */
4573         code = 5;
4574         goto do_fault;
4575     }
4576     if (domain_prot == 0 || domain_prot == 2) {
4577         if (type == 2)
4578             code = 9; /* Section domain fault.  */
4579         else
4580             code = 11; /* Page domain fault.  */
4581         goto do_fault;
4582     }
4583     if (type == 2) {
4584         /* 1Mb section.  */
4585         phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
4586         ap = (desc >> 10) & 3;
4587         code = 13;
4588         *page_size = 1024 * 1024;
4589     } else {
4590         /* Lookup l2 entry.  */
4591         if (type == 1) {
4592             /* Coarse pagetable.  */
4593             table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
4594         } else {
4595             /* Fine pagetable.  */
4596             table = (desc & 0xfffff000) | ((address >> 8) & 0xffc);
4597         }
4598         desc = ldl_phys(cs->as, table);
4599         switch (desc & 3) {
4600         case 0: /* Page translation fault.  */
4601             code = 7;
4602             goto do_fault;
4603         case 1: /* 64k page.  */
4604             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
4605             ap = (desc >> (4 + ((address >> 13) & 6))) & 3;
4606             *page_size = 0x10000;
4607             break;
4608         case 2: /* 4k page.  */
4609             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
4610             ap = (desc >> (4 + ((address >> 9) & 6))) & 3;
4611             *page_size = 0x1000;
4612             break;
4613         case 3: /* 1k page.  */
4614             if (type == 1) {
4615                 if (arm_feature(env, ARM_FEATURE_XSCALE)) {
4616                     phys_addr = (desc & 0xfffff000) | (address & 0xfff);
4617                 } else {
4618                     /* Page translation fault.  */
4619                     code = 7;
4620                     goto do_fault;
4621                 }
4622             } else {
4623                 phys_addr = (desc & 0xfffffc00) | (address & 0x3ff);
4624             }
4625             ap = (desc >> 4) & 3;
4626             *page_size = 0x400;
4627             break;
4628         default:
4629             /* Never happens, but compiler isn't smart enough to tell.  */
4630             abort();
4631         }
4632         code = 15;
4633     }
4634     *prot = check_ap(env, ap, domain_prot, access_type, is_user);
4635     if (!*prot) {
4636         /* Access permission fault.  */
4637         goto do_fault;
4638     }
4639     *prot |= PAGE_EXEC;
4640     *phys_ptr = phys_addr;
4641     return 0;
4642 do_fault:
4643     return code | (domain << 4);
4644 }
4645
4646 static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
4647                             int is_user, hwaddr *phys_ptr,
4648                             int *prot, target_ulong *page_size)
4649 {
4650     CPUState *cs = CPU(arm_env_get_cpu(env));
4651     int code;
4652     uint32_t table;
4653     uint32_t desc;
4654     uint32_t xn;
4655     uint32_t pxn = 0;
4656     int type;
4657     int ap;
4658     int domain = 0;
4659     int domain_prot;
4660     hwaddr phys_addr;
4661
4662     /* Pagetable walk.  */
4663     /* Lookup l1 descriptor.  */
4664     if (!get_level1_table_address(env, &table, address)) {
4665         /* Section translation fault if page walk is disabled by PD0 or PD1 */
4666         code = 5;
4667         goto do_fault;
4668     }
4669     desc = ldl_phys(cs->as, table);
4670     type = (desc & 3);
4671     if (type == 0 || (type == 3 && !arm_feature(env, ARM_FEATURE_PXN))) {
4672         /* Section translation fault, or attempt to use the encoding
4673          * which is Reserved on implementations without PXN.
4674          */
4675         code = 5;
4676         goto do_fault;
4677     }
4678     if ((type == 1) || !(desc & (1 << 18))) {
4679         /* Page or Section.  */
4680         domain = (desc >> 5) & 0x0f;
4681     }
4682     domain_prot = (A32_BANKED_CURRENT_REG_GET(env, dacr) >> (domain * 2)) & 3;
4683     if (domain_prot == 0 || domain_prot == 2) {
4684         if (type != 1) {
4685             code = 9; /* Section domain fault.  */
4686         } else {
4687             code = 11; /* Page domain fault.  */
4688         }
4689         goto do_fault;
4690     }
4691     if (type != 1) {
4692         if (desc & (1 << 18)) {
4693             /* Supersection.  */
4694             phys_addr = (desc & 0xff000000) | (address & 0x00ffffff);
4695             *page_size = 0x1000000;
4696         } else {
4697             /* Section.  */
4698             phys_addr = (desc & 0xfff00000) | (address & 0x000fffff);
4699             *page_size = 0x100000;
4700         }
4701         ap = ((desc >> 10) & 3) | ((desc >> 13) & 4);
4702         xn = desc & (1 << 4);
4703         pxn = desc & 1;
4704         code = 13;
4705     } else {
4706         if (arm_feature(env, ARM_FEATURE_PXN)) {
4707             pxn = (desc >> 2) & 1;
4708         }
4709         /* Lookup l2 entry.  */
4710         table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc);
4711         desc = ldl_phys(cs->as, table);
4712         ap = ((desc >> 4) & 3) | ((desc >> 7) & 4);
4713         switch (desc & 3) {
4714         case 0: /* Page translation fault.  */
4715             code = 7;
4716             goto do_fault;
4717         case 1: /* 64k page.  */
4718             phys_addr = (desc & 0xffff0000) | (address & 0xffff);
4719             xn = desc & (1 << 15);
4720             *page_size = 0x10000;
4721             break;
4722         case 2: case 3: /* 4k page.  */
4723             phys_addr = (desc & 0xfffff000) | (address & 0xfff);
4724             xn = desc & 1;
4725             *page_size = 0x1000;
4726             break;
4727         default:
4728             /* Never happens, but compiler isn't smart enough to tell.  */
4729             abort();
4730         }
4731         code = 15;
4732     }
4733     if (domain_prot == 3) {
4734         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
4735     } else {
4736         if (pxn && !is_user) {
4737             xn = 1;
4738         }
4739         if (xn && access_type == 2)
4740             goto do_fault;
4741
4742         /* The simplified model uses AP[0] as an access control bit.  */
4743         if ((A32_BANKED_CURRENT_REG_GET(env, sctlr) & SCTLR_AFE)
4744                 && (ap & 1) == 0) {
4745             /* Access flag fault.  */
4746             code = (code == 15) ? 6 : 3;
4747             goto do_fault;
4748         }
4749         *prot = check_ap(env, ap, domain_prot, access_type, is_user);
4750         if (!*prot) {
4751             /* Access permission fault.  */
4752             goto do_fault;
4753         }
4754         if (!xn) {
4755             *prot |= PAGE_EXEC;
4756         }
4757     }
4758     *phys_ptr = phys_addr;
4759     return 0;
4760 do_fault:
4761     return code | (domain << 4);
4762 }
4763
4764 /* Fault type for long-descriptor MMU fault reporting; this corresponds
4765  * to bits [5..2] in the STATUS field in long-format DFSR/IFSR.
4766  */
4767 typedef enum {
4768     translation_fault = 1,
4769     access_fault = 2,
4770     permission_fault = 3,
4771 } MMUFaultType;
4772
4773 static int get_phys_addr_lpae(CPUARMState *env, target_ulong address,
4774                               int access_type, int is_user,
4775                               hwaddr *phys_ptr, int *prot,
4776                               target_ulong *page_size_ptr)
4777 {
4778     CPUState *cs = CPU(arm_env_get_cpu(env));
4779     /* Read an LPAE long-descriptor translation table. */
4780     MMUFaultType fault_type = translation_fault;
4781     uint32_t level = 1;
4782     uint32_t epd;
4783     int32_t tsz;
4784     uint32_t tg;
4785     uint64_t ttbr;
4786     int ttbr_select;
4787     hwaddr descaddr, descmask;
4788     uint32_t tableattrs;
4789     target_ulong page_size;
4790     uint32_t attrs;
4791     int32_t granule_sz = 9;
4792     int32_t va_size = 32;
4793     int32_t tbi = 0;
4794     TCR *tcr = &env->cp15.tcr_el[arm_is_secure(env) ? 3 : 1];
4795
4796     if (arm_el_is_aa64(env, 1)) {
4797         va_size = 64;
4798         if (extract64(address, 55, 1))
4799             tbi = extract64(tcr->raw_tcr, 38, 1);
4800         else
4801             tbi = extract64(tcr->raw_tcr, 37, 1);
4802         tbi *= 8;
4803     }
4804
4805     /* Determine whether this address is in the region controlled by
4806      * TTBR0 or TTBR1 (or if it is in neither region and should fault).
4807      * This is a Non-secure PL0/1 stage 1 translation, so controlled by
4808      * TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
4809      */
4810     uint32_t t0sz = extract32(tcr->raw_tcr, 0, 6);
4811     if (arm_el_is_aa64(env, 1)) {
4812         t0sz = MIN(t0sz, 39);
4813         t0sz = MAX(t0sz, 16);
4814     }
4815     uint32_t t1sz = extract32(tcr->raw_tcr, 16, 6);
4816     if (arm_el_is_aa64(env, 1)) {
4817         t1sz = MIN(t1sz, 39);
4818         t1sz = MAX(t1sz, 16);
4819     }
4820     if (t0sz && !extract64(address, va_size - t0sz, t0sz - tbi)) {
4821         /* there is a ttbr0 region and we are in it (high bits all zero) */
4822         ttbr_select = 0;
4823     } else if (t1sz && !extract64(~address, va_size - t1sz, t1sz - tbi)) {
4824         /* there is a ttbr1 region and we are in it (high bits all one) */
4825         ttbr_select = 1;
4826     } else if (!t0sz) {
4827         /* ttbr0 region is "everything not in the ttbr1 region" */
4828         ttbr_select = 0;
4829     } else if (!t1sz) {
4830         /* ttbr1 region is "everything not in the ttbr0 region" */
4831         ttbr_select = 1;
4832     } else {
4833         /* in the gap between the two regions, this is a Translation fault */
4834         fault_type = translation_fault;
4835         goto do_fault;
4836     }
4837
4838     /* Note that QEMU ignores shareability and cacheability attributes,
4839      * so we don't need to do anything with the SH, ORGN, IRGN fields
4840      * in the TTBCR.  Similarly, TTBCR:A1 selects whether we get the
4841      * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently
4842      * implement any ASID-like capability so we can ignore it (instead
4843      * we will always flush the TLB any time the ASID is changed).
4844      */
4845     if (ttbr_select == 0) {
4846         ttbr = A32_BANKED_CURRENT_REG_GET(env, ttbr0);
4847         epd = extract32(tcr->raw_tcr, 7, 1);
4848         tsz = t0sz;
4849
4850         tg = extract32(tcr->raw_tcr, 14, 2);
4851         if (tg == 1) { /* 64KB pages */
4852             granule_sz = 13;
4853         }
4854         if (tg == 2) { /* 16KB pages */
4855             granule_sz = 11;
4856         }
4857     } else {
4858         ttbr = A32_BANKED_CURRENT_REG_GET(env, ttbr1);
4859         epd = extract32(tcr->raw_tcr, 23, 1);
4860         tsz = t1sz;
4861
4862         tg = extract32(tcr->raw_tcr, 30, 2);
4863         if (tg == 3)  { /* 64KB pages */
4864             granule_sz = 13;
4865         }
4866         if (tg == 1) { /* 16KB pages */
4867             granule_sz = 11;
4868         }
4869     }
4870
4871     if (epd) {
4872         /* Translation table walk disabled => Translation fault on TLB miss */
4873         goto do_fault;
4874     }
4875
4876     /* The starting level depends on the virtual address size (which can be
4877      * up to 48 bits) and the translation granule size. It indicates the number
4878      * of strides (granule_sz bits at a time) needed to consume the bits
4879      * of the input address. In the pseudocode this is:
4880      *  level = 4 - RoundUp((inputsize - grainsize) / stride)
4881      * where their 'inputsize' is our 'va_size - tsz', 'grainsize' is
4882      * our 'granule_sz + 3' and 'stride' is our 'granule_sz'.
4883      * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying:
4884      *     = 4 - (va_size - tsz - granule_sz - 3 + granule_sz - 1) / granule_sz
4885      *     = 4 - (va_size - tsz - 4) / granule_sz;
4886      */
4887     level = 4 - (va_size - tsz - 4) / granule_sz;
4888
4889     /* Clear the vaddr bits which aren't part of the within-region address,
4890      * so that we don't have to special case things when calculating the
4891      * first descriptor address.
4892      */
4893     if (tsz) {
4894         address &= (1ULL << (va_size - tsz)) - 1;
4895     }
4896
4897     descmask = (1ULL << (granule_sz + 3)) - 1;
4898
4899     /* Now we can extract the actual base address from the TTBR */
4900     descaddr = extract64(ttbr, 0, 48);
4901     descaddr &= ~((1ULL << (va_size - tsz - (granule_sz * (4 - level)))) - 1);
4902
4903     tableattrs = 0;
4904     for (;;) {
4905         uint64_t descriptor;
4906
4907         descaddr |= (address >> (granule_sz * (4 - level))) & descmask;
4908         descaddr &= ~7ULL;
4909         descriptor = ldq_phys(cs->as, descaddr);
4910         if (!(descriptor & 1) ||
4911             (!(descriptor & 2) && (level == 3))) {
4912             /* Invalid, or the Reserved level 3 encoding */
4913             goto do_fault;
4914         }
4915         descaddr = descriptor & 0xfffffff000ULL;
4916
4917         if ((descriptor & 2) && (level < 3)) {
4918             /* Table entry. The top five bits are attributes which  may
4919              * propagate down through lower levels of the table (and
4920              * which are all arranged so that 0 means "no effect", so
4921              * we can gather them up by ORing in the bits at each level).
4922              */
4923             tableattrs |= extract64(descriptor, 59, 5);
4924             level++;
4925             continue;
4926         }
4927         /* Block entry at level 1 or 2, or page entry at level 3.
4928          * These are basically the same thing, although the number
4929          * of bits we pull in from the vaddr varies.
4930          */
4931         page_size = (1ULL << ((granule_sz * (4 - level)) + 3));
4932         descaddr |= (address & (page_size - 1));
4933         /* Extract attributes from the descriptor and merge with table attrs */
4934         attrs = extract64(descriptor, 2, 10)
4935             | (extract64(descriptor, 52, 12) << 10);
4936         attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
4937         attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
4938         /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
4939          * means "force PL1 access only", which means forcing AP[1] to 0.
4940          */
4941         if (extract32(tableattrs, 2, 1)) {
4942             attrs &= ~(1 << 4);
4943         }
4944         /* Since we're always in the Non-secure state, NSTable is ignored. */
4945         break;
4946     }
4947     /* Here descaddr is the final physical address, and attributes
4948      * are all in attrs.
4949      */
4950     fault_type = access_fault;
4951     if ((attrs & (1 << 8)) == 0) {
4952         /* Access flag */
4953         goto do_fault;
4954     }
4955     fault_type = permission_fault;
4956     if (is_user && !(attrs & (1 << 4))) {
4957         /* Unprivileged access not enabled */
4958         goto do_fault;
4959     }
4960     *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
4961     if ((arm_feature(env, ARM_FEATURE_V8) && is_user && (attrs & (1 << 12))) ||
4962         (!arm_feature(env, ARM_FEATURE_V8) && (attrs & (1 << 12))) ||
4963         (!is_user && (attrs & (1 << 11)))) {
4964         /* XN/UXN or PXN. Since we only implement EL0/EL1 we unconditionally
4965          * treat XN/UXN as UXN for v8.
4966          */
4967         if (access_type == 2) {
4968             goto do_fault;
4969         }
4970         *prot &= ~PAGE_EXEC;
4971     }
4972     if (attrs & (1 << 5)) {
4973         /* Write access forbidden */
4974         if (access_type == 1) {
4975             goto do_fault;
4976         }
4977         *prot &= ~PAGE_WRITE;
4978     }
4979
4980     *phys_ptr = descaddr;
4981     *page_size_ptr = page_size;
4982     return 0;
4983
4984 do_fault:
4985     /* Long-descriptor format IFSR/DFSR value */
4986     return (1 << 9) | (fault_type << 2) | level;
4987 }
4988
4989 static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
4990                              int access_type, int is_user,
4991                              hwaddr *phys_ptr, int *prot)
4992 {
4993     int n;
4994     uint32_t mask;
4995     uint32_t base;
4996
4997     *phys_ptr = address;
4998     for (n = 7; n >= 0; n--) {
4999         base = env->cp15.c6_region[n];
5000         if ((base & 1) == 0)
5001             continue;
5002         mask = 1 << ((base >> 1) & 0x1f);
5003         /* Keep this shift separate from the above to avoid an
5004            (undefined) << 32.  */
5005         mask = (mask << 1) - 1;
5006         if (((base ^ address) & ~mask) == 0)
5007             break;
5008     }
5009     if (n < 0)
5010         return 2;
5011
5012     if (access_type == 2) {
5013         mask = env->cp15.pmsav5_insn_ap;
5014     } else {
5015         mask = env->cp15.pmsav5_data_ap;
5016     }
5017     mask = (mask >> (n * 4)) & 0xf;
5018     switch (mask) {
5019     case 0:
5020         return 1;
5021     case 1:
5022         if (is_user)
5023           return 1;
5024         *prot = PAGE_READ | PAGE_WRITE;
5025         break;
5026     case 2:
5027         *prot = PAGE_READ;
5028         if (!is_user)
5029             *prot |= PAGE_WRITE;
5030         break;
5031     case 3:
5032         *prot = PAGE_READ | PAGE_WRITE;
5033         break;
5034     case 5:
5035         if (is_user)
5036             return 1;
5037         *prot = PAGE_READ;
5038         break;
5039     case 6:
5040         *prot = PAGE_READ;
5041         break;
5042     default:
5043         /* Bad permission.  */
5044         return 1;
5045     }
5046     *prot |= PAGE_EXEC;
5047     return 0;
5048 }
5049
5050 /* get_phys_addr - get the physical address for this virtual address
5051  *
5052  * Find the physical address corresponding to the given virtual address,
5053  * by doing a translation table walk on MMU based systems or using the
5054  * MPU state on MPU based systems.
5055  *
5056  * Returns 0 if the translation was successful. Otherwise, phys_ptr,
5057  * prot and page_size are not filled in, and the return value provides
5058  * information on why the translation aborted, in the format of a
5059  * DFSR/IFSR fault register, with the following caveats:
5060  *  * we honour the short vs long DFSR format differences.
5061  *  * the WnR bit is never set (the caller must do this).
5062  *  * for MPU based systems we don't bother to return a full FSR format
5063  *    value.
5064  *
5065  * @env: CPUARMState
5066  * @address: virtual address to get physical address for
5067  * @access_type: 0 for read, 1 for write, 2 for execute
5068  * @is_user: 0 for privileged access, 1 for user
5069  * @phys_ptr: set to the physical address corresponding to the virtual address
5070  * @prot: set to the permissions for the page containing phys_ptr
5071  * @page_size: set to the size of the page containing phys_ptr
5072  */
5073 static inline int get_phys_addr(CPUARMState *env, target_ulong address,
5074                                 int access_type, int is_user,
5075                                 hwaddr *phys_ptr, int *prot,
5076                                 target_ulong *page_size)
5077 {
5078     /* This is not entirely correct as get_phys_addr() can also be called
5079      * from ats_write() for an address translation of a specific regime.
5080      */
5081     uint32_t sctlr = A32_BANKED_CURRENT_REG_GET(env, sctlr);
5082
5083     /* Fast Context Switch Extension.  */
5084     if (address < 0x02000000) {
5085         address += A32_BANKED_CURRENT_REG_GET(env, fcseidr);
5086     }
5087
5088     if ((sctlr & SCTLR_M) == 0) {
5089         /* MMU/MPU disabled.  */
5090         *phys_ptr = address;
5091         *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
5092         *page_size = TARGET_PAGE_SIZE;
5093         return 0;
5094     } else if (arm_feature(env, ARM_FEATURE_MPU)) {
5095         *page_size = TARGET_PAGE_SIZE;
5096         return get_phys_addr_mpu(env, address, access_type, is_user, phys_ptr,
5097                                  prot);
5098     } else if (extended_addresses_enabled(env)) {
5099         return get_phys_addr_lpae(env, address, access_type, is_user, phys_ptr,
5100                                   prot, page_size);
5101     } else if (sctlr & SCTLR_XP) {
5102         return get_phys_addr_v6(env, address, access_type, is_user, phys_ptr,
5103                                 prot, page_size);
5104     } else {
5105         return get_phys_addr_v5(env, address, access_type, is_user, phys_ptr,
5106                                 prot, page_size);
5107     }
5108 }
5109
5110 int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
5111                              int access_type, int mmu_idx)
5112 {
5113     ARMCPU *cpu = ARM_CPU(cs);
5114     CPUARMState *env = &cpu->env;
5115     hwaddr phys_addr;
5116     target_ulong page_size;
5117     int prot;
5118     int ret, is_user;
5119     uint32_t syn;
5120     bool same_el = (arm_current_el(env) != 0);
5121
5122     is_user = mmu_idx == MMU_USER_IDX;
5123     ret = get_phys_addr(env, address, access_type, is_user, &phys_addr, &prot,
5124                         &page_size);
5125     if (ret == 0) {
5126         /* Map a single [sub]page.  */
5127         phys_addr &= TARGET_PAGE_MASK;
5128         address &= TARGET_PAGE_MASK;
5129         tlb_set_page(cs, address, phys_addr, prot, mmu_idx, page_size);
5130         return 0;
5131     }
5132
5133     /* AArch64 syndrome does not have an LPAE bit */
5134     syn = ret & ~(1 << 9);
5135
5136     /* For insn and data aborts we assume there is no instruction syndrome
5137      * information; this is always true for exceptions reported to EL1.
5138      */
5139     if (access_type == 2) {
5140         syn = syn_insn_abort(same_el, 0, 0, syn);
5141         cs->exception_index = EXCP_PREFETCH_ABORT;
5142     } else {
5143         syn = syn_data_abort(same_el, 0, 0, 0, access_type == 1, syn);
5144         if (access_type == 1 && arm_feature(env, ARM_FEATURE_V6)) {
5145             ret |= (1 << 11);
5146         }
5147         cs->exception_index = EXCP_DATA_ABORT;
5148     }
5149
5150     env->exception.syndrome = syn;
5151     env->exception.vaddress = address;
5152     env->exception.fsr = ret;
5153     return 1;
5154 }
5155
5156 hwaddr arm_cpu_get_phys_page_debug(CPUState *cs, vaddr addr)
5157 {
5158     ARMCPU *cpu = ARM_CPU(cs);
5159     hwaddr phys_addr;
5160     target_ulong page_size;
5161     int prot;
5162     int ret;
5163
5164     ret = get_phys_addr(&cpu->env, addr, 0, 0, &phys_addr, &prot, &page_size);
5165
5166     if (ret != 0) {
5167         return -1;
5168     }
5169
5170     return phys_addr;
5171 }
5172
5173 void HELPER(set_r13_banked)(CPUARMState *env, uint32_t mode, uint32_t val)
5174 {
5175     if ((env->uncached_cpsr & CPSR_M) == mode) {
5176         env->regs[13] = val;
5177     } else {
5178         env->banked_r13[bank_number(mode)] = val;
5179     }
5180 }
5181
5182 uint32_t HELPER(get_r13_banked)(CPUARMState *env, uint32_t mode)
5183 {
5184     if ((env->uncached_cpsr & CPSR_M) == mode) {
5185         return env->regs[13];
5186     } else {
5187         return env->banked_r13[bank_number(mode)];
5188     }
5189 }
5190
5191 uint32_t HELPER(v7m_mrs)(CPUARMState *env, uint32_t reg)
5192 {
5193     ARMCPU *cpu = arm_env_get_cpu(env);
5194
5195     switch (reg) {
5196     case 0: /* APSR */
5197         return xpsr_read(env) & 0xf8000000;
5198     case 1: /* IAPSR */
5199         return xpsr_read(env) & 0xf80001ff;
5200     case 2: /* EAPSR */
5201         return xpsr_read(env) & 0xff00fc00;
5202     case 3: /* xPSR */
5203         return xpsr_read(env) & 0xff00fdff;
5204     case 5: /* IPSR */
5205         return xpsr_read(env) & 0x000001ff;
5206     case 6: /* EPSR */
5207         return xpsr_read(env) & 0x0700fc00;
5208     case 7: /* IEPSR */
5209         return xpsr_read(env) & 0x0700edff;
5210     case 8: /* MSP */
5211         return env->v7m.current_sp ? env->v7m.other_sp : env->regs[13];
5212     case 9: /* PSP */
5213         return env->v7m.current_sp ? env->regs[13] : env->v7m.other_sp;
5214     case 16: /* PRIMASK */
5215         return (env->daif & PSTATE_I) != 0;
5216     case 17: /* BASEPRI */
5217     case 18: /* BASEPRI_MAX */
5218         return env->v7m.basepri;
5219     case 19: /* FAULTMASK */
5220         return (env->daif & PSTATE_F) != 0;
5221     case 20: /* CONTROL */
5222         return env->v7m.control;
5223     default:
5224         /* ??? For debugging only.  */
5225         cpu_abort(CPU(cpu), "Unimplemented system register read (%d)\n", reg);
5226         return 0;
5227     }
5228 }
5229
5230 void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
5231 {
5232     ARMCPU *cpu = arm_env_get_cpu(env);
5233
5234     switch (reg) {
5235     case 0: /* APSR */
5236         xpsr_write(env, val, 0xf8000000);
5237         break;
5238     case 1: /* IAPSR */
5239         xpsr_write(env, val, 0xf8000000);
5240         break;
5241     case 2: /* EAPSR */
5242         xpsr_write(env, val, 0xfe00fc00);
5243         break;
5244     case 3: /* xPSR */
5245         xpsr_write(env, val, 0xfe00fc00);
5246         break;
5247     case 5: /* IPSR */
5248         /* IPSR bits are readonly.  */
5249         break;
5250     case 6: /* EPSR */
5251         xpsr_write(env, val, 0x0600fc00);
5252         break;
5253     case 7: /* IEPSR */
5254         xpsr_write(env, val, 0x0600fc00);
5255         break;
5256     case 8: /* MSP */
5257         if (env->v7m.current_sp)
5258             env->v7m.other_sp = val;
5259         else
5260             env->regs[13] = val;
5261         break;
5262     case 9: /* PSP */
5263         if (env->v7m.current_sp)
5264             env->regs[13] = val;
5265         else
5266             env->v7m.other_sp = val;
5267         break;
5268     case 16: /* PRIMASK */
5269         if (val & 1) {
5270             env->daif |= PSTATE_I;
5271         } else {
5272             env->daif &= ~PSTATE_I;
5273         }
5274         break;
5275     case 17: /* BASEPRI */
5276         env->v7m.basepri = val & 0xff;
5277         break;
5278     case 18: /* BASEPRI_MAX */
5279         val &= 0xff;
5280         if (val != 0 && (val < env->v7m.basepri || env->v7m.basepri == 0))
5281             env->v7m.basepri = val;
5282         break;
5283     case 19: /* FAULTMASK */
5284         if (val & 1) {
5285             env->daif |= PSTATE_F;
5286         } else {
5287             env->daif &= ~PSTATE_F;
5288         }
5289         break;
5290     case 20: /* CONTROL */
5291         env->v7m.control = val & 3;
5292         switch_v7m_sp(env, (val & 2) != 0);
5293         break;
5294     default:
5295         /* ??? For debugging only.  */
5296         cpu_abort(CPU(cpu), "Unimplemented system register write (%d)\n", reg);
5297         return;
5298     }
5299 }
5300
5301 #endif
5302
5303 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
5304 {
5305     /* Implement DC ZVA, which zeroes a fixed-length block of memory.
5306      * Note that we do not implement the (architecturally mandated)
5307      * alignment fault for attempts to use this on Device memory
5308      * (which matches the usual QEMU behaviour of not implementing either
5309      * alignment faults or any memory attribute handling).
5310      */
5311
5312     ARMCPU *cpu = arm_env_get_cpu(env);
5313     uint64_t blocklen = 4 << cpu->dcz_blocksize;
5314     uint64_t vaddr = vaddr_in & ~(blocklen - 1);
5315
5316 #ifndef CONFIG_USER_ONLY
5317     {
5318         /* Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than
5319          * the block size so we might have to do more than one TLB lookup.
5320          * We know that in fact for any v8 CPU the page size is at least 4K
5321          * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only
5322          * 1K as an artefact of legacy v5 subpage support being present in the
5323          * same QEMU executable.
5324          */
5325         int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
5326         void *hostaddr[maxidx];
5327         int try, i;
5328
5329         for (try = 0; try < 2; try++) {
5330
5331             for (i = 0; i < maxidx; i++) {
5332                 hostaddr[i] = tlb_vaddr_to_host(env,
5333                                                 vaddr + TARGET_PAGE_SIZE * i,
5334                                                 1, cpu_mmu_index(env));
5335                 if (!hostaddr[i]) {
5336                     break;
5337                 }
5338             }
5339             if (i == maxidx) {
5340                 /* If it's all in the TLB it's fair game for just writing to;
5341                  * we know we don't need to update dirty status, etc.
5342                  */
5343                 for (i = 0; i < maxidx - 1; i++) {
5344                     memset(hostaddr[i], 0, TARGET_PAGE_SIZE);
5345                 }
5346                 memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE));
5347                 return;
5348             }
5349             /* OK, try a store and see if we can populate the tlb. This
5350              * might cause an exception if the memory isn't writable,
5351              * in which case we will longjmp out of here. We must for
5352              * this purpose use the actual register value passed to us
5353              * so that we get the fault address right.
5354              */
5355             helper_ret_stb_mmu(env, vaddr_in, 0, cpu_mmu_index(env), GETRA());
5356             /* Now we can populate the other TLB entries, if any */
5357             for (i = 0; i < maxidx; i++) {
5358                 uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
5359                 if (va != (vaddr_in & TARGET_PAGE_MASK)) {
5360                     helper_ret_stb_mmu(env, va, 0, cpu_mmu_index(env), GETRA());
5361                 }
5362             }
5363         }
5364
5365         /* Slow path (probably attempt to do this to an I/O device or
5366          * similar, or clearing of a block of code we have translations
5367          * cached for). Just do a series of byte writes as the architecture
5368          * demands. It's not worth trying to use a cpu_physical_memory_map(),
5369          * memset(), unmap() sequence here because:
5370          *  + we'd need to account for the blocksize being larger than a page
5371          *  + the direct-RAM access case is almost always going to be dealt
5372          *    with in the fastpath code above, so there's no speed benefit
5373          *  + we would have to deal with the map returning NULL because the
5374          *    bounce buffer was in use
5375          */
5376         for (i = 0; i < blocklen; i++) {
5377             helper_ret_stb_mmu(env, vaddr + i, 0, cpu_mmu_index(env), GETRA());
5378         }
5379     }
5380 #else
5381     memset(g2h(vaddr), 0, blocklen);
5382 #endif
5383 }
5384
5385 /* Note that signed overflow is undefined in C.  The following routines are
5386    careful to use unsigned types where modulo arithmetic is required.
5387    Failure to do so _will_ break on newer gcc.  */
5388
5389 /* Signed saturating arithmetic.  */
5390
5391 /* Perform 16-bit signed saturating addition.  */
5392 static inline uint16_t add16_sat(uint16_t a, uint16_t b)
5393 {
5394     uint16_t res;
5395
5396     res = a + b;
5397     if (((res ^ a) & 0x8000) && !((a ^ b) & 0x8000)) {
5398         if (a & 0x8000)
5399             res = 0x8000;
5400         else
5401             res = 0x7fff;
5402     }
5403     return res;
5404 }
5405
5406 /* Perform 8-bit signed saturating addition.  */
5407 static inline uint8_t add8_sat(uint8_t a, uint8_t b)
5408 {
5409     uint8_t res;
5410
5411     res = a + b;
5412     if (((res ^ a) & 0x80) && !((a ^ b) & 0x80)) {
5413         if (a & 0x80)
5414             res = 0x80;
5415         else
5416             res = 0x7f;
5417     }
5418     return res;
5419 }
5420
5421 /* Perform 16-bit signed saturating subtraction.  */
5422 static inline uint16_t sub16_sat(uint16_t a, uint16_t b)
5423 {
5424     uint16_t res;
5425
5426     res = a - b;
5427     if (((res ^ a) & 0x8000) && ((a ^ b) & 0x8000)) {
5428         if (a & 0x8000)
5429             res = 0x8000;
5430         else
5431             res = 0x7fff;
5432     }
5433     return res;
5434 }
5435
5436 /* Perform 8-bit signed saturating subtraction.  */
5437 static inline uint8_t sub8_sat(uint8_t a, uint8_t b)
5438 {
5439     uint8_t res;
5440
5441     res = a - b;
5442     if (((res ^ a) & 0x80) && ((a ^ b) & 0x80)) {
5443         if (a & 0x80)
5444             res = 0x80;
5445         else
5446             res = 0x7f;
5447     }
5448     return res;
5449 }
5450
5451 #define ADD16(a, b, n) RESULT(add16_sat(a, b), n, 16);
5452 #define SUB16(a, b, n) RESULT(sub16_sat(a, b), n, 16);
5453 #define ADD8(a, b, n)  RESULT(add8_sat(a, b), n, 8);
5454 #define SUB8(a, b, n)  RESULT(sub8_sat(a, b), n, 8);
5455 #define PFX q
5456
5457 #include "op_addsub.h"
5458
5459 /* Unsigned saturating arithmetic.  */
5460 static inline uint16_t add16_usat(uint16_t a, uint16_t b)
5461 {
5462     uint16_t res;
5463     res = a + b;
5464     if (res < a)
5465         res = 0xffff;
5466     return res;
5467 }
5468
5469 static inline uint16_t sub16_usat(uint16_t a, uint16_t b)
5470 {
5471     if (a > b)
5472         return a - b;
5473     else
5474         return 0;
5475 }
5476
5477 static inline uint8_t add8_usat(uint8_t a, uint8_t b)
5478 {
5479     uint8_t res;
5480     res = a + b;
5481     if (res < a)
5482         res = 0xff;
5483     return res;
5484 }
5485
5486 static inline uint8_t sub8_usat(uint8_t a, uint8_t b)
5487 {
5488     if (a > b)
5489         return a - b;
5490     else
5491         return 0;
5492 }
5493
5494 #define ADD16(a, b, n) RESULT(add16_usat(a, b), n, 16);
5495 #define SUB16(a, b, n) RESULT(sub16_usat(a, b), n, 16);
5496 #define ADD8(a, b, n)  RESULT(add8_usat(a, b), n, 8);
5497 #define SUB8(a, b, n)  RESULT(sub8_usat(a, b), n, 8);
5498 #define PFX uq
5499
5500 #include "op_addsub.h"
5501
5502 /* Signed modulo arithmetic.  */
5503 #define SARITH16(a, b, n, op) do { \
5504     int32_t sum; \
5505     sum = (int32_t)(int16_t)(a) op (int32_t)(int16_t)(b); \
5506     RESULT(sum, n, 16); \
5507     if (sum >= 0) \
5508         ge |= 3 << (n * 2); \
5509     } while(0)
5510
5511 #define SARITH8(a, b, n, op) do { \
5512     int32_t sum; \
5513     sum = (int32_t)(int8_t)(a) op (int32_t)(int8_t)(b); \
5514     RESULT(sum, n, 8); \
5515     if (sum >= 0) \
5516         ge |= 1 << n; \
5517     } while(0)
5518
5519
5520 #define ADD16(a, b, n) SARITH16(a, b, n, +)
5521 #define SUB16(a, b, n) SARITH16(a, b, n, -)
5522 #define ADD8(a, b, n)  SARITH8(a, b, n, +)
5523 #define SUB8(a, b, n)  SARITH8(a, b, n, -)
5524 #define PFX s
5525 #define ARITH_GE
5526
5527 #include "op_addsub.h"
5528
5529 /* Unsigned modulo arithmetic.  */
5530 #define ADD16(a, b, n) do { \
5531     uint32_t sum; \
5532     sum = (uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b); \
5533     RESULT(sum, n, 16); \
5534     if ((sum >> 16) == 1) \
5535         ge |= 3 << (n * 2); \
5536     } while(0)
5537
5538 #define ADD8(a, b, n) do { \
5539     uint32_t sum; \
5540     sum = (uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b); \
5541     RESULT(sum, n, 8); \
5542     if ((sum >> 8) == 1) \
5543         ge |= 1 << n; \
5544     } while(0)
5545
5546 #define SUB16(a, b, n) do { \
5547     uint32_t sum; \
5548     sum = (uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b); \
5549     RESULT(sum, n, 16); \
5550     if ((sum >> 16) == 0) \
5551         ge |= 3 << (n * 2); \
5552     } while(0)
5553
5554 #define SUB8(a, b, n) do { \
5555     uint32_t sum; \
5556     sum = (uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b); \
5557     RESULT(sum, n, 8); \
5558     if ((sum >> 8) == 0) \
5559         ge |= 1 << n; \
5560     } while(0)
5561
5562 #define PFX u
5563 #define ARITH_GE
5564
5565 #include "op_addsub.h"
5566
5567 /* Halved signed arithmetic.  */
5568 #define ADD16(a, b, n) \
5569   RESULT(((int32_t)(int16_t)(a) + (int32_t)(int16_t)(b)) >> 1, n, 16)
5570 #define SUB16(a, b, n) \
5571   RESULT(((int32_t)(int16_t)(a) - (int32_t)(int16_t)(b)) >> 1, n, 16)
5572 #define ADD8(a, b, n) \
5573   RESULT(((int32_t)(int8_t)(a) + (int32_t)(int8_t)(b)) >> 1, n, 8)
5574 #define SUB8(a, b, n) \
5575   RESULT(((int32_t)(int8_t)(a) - (int32_t)(int8_t)(b)) >> 1, n, 8)
5576 #define PFX sh
5577
5578 #include "op_addsub.h"
5579
5580 /* Halved unsigned arithmetic.  */
5581 #define ADD16(a, b, n) \
5582   RESULT(((uint32_t)(uint16_t)(a) + (uint32_t)(uint16_t)(b)) >> 1, n, 16)
5583 #define SUB16(a, b, n) \
5584   RESULT(((uint32_t)(uint16_t)(a) - (uint32_t)(uint16_t)(b)) >> 1, n, 16)
5585 #define ADD8(a, b, n) \
5586   RESULT(((uint32_t)(uint8_t)(a) + (uint32_t)(uint8_t)(b)) >> 1, n, 8)
5587 #define SUB8(a, b, n) \
5588   RESULT(((uint32_t)(uint8_t)(a) - (uint32_t)(uint8_t)(b)) >> 1, n, 8)
5589 #define PFX uh
5590
5591 #include "op_addsub.h"
5592
5593 static inline uint8_t do_usad(uint8_t a, uint8_t b)
5594 {
5595     if (a > b)
5596         return a - b;
5597     else
5598         return b - a;
5599 }
5600
5601 /* Unsigned sum of absolute byte differences.  */
5602 uint32_t HELPER(usad8)(uint32_t a, uint32_t b)
5603 {
5604     uint32_t sum;
5605     sum = do_usad(a, b);
5606     sum += do_usad(a >> 8, b >> 8);
5607     sum += do_usad(a >> 16, b >>16);
5608     sum += do_usad(a >> 24, b >> 24);
5609     return sum;
5610 }
5611
5612 /* For ARMv6 SEL instruction.  */
5613 uint32_t HELPER(sel_flags)(uint32_t flags, uint32_t a, uint32_t b)
5614 {
5615     uint32_t mask;
5616
5617     mask = 0;
5618     if (flags & 1)
5619         mask |= 0xff;
5620     if (flags & 2)
5621         mask |= 0xff00;
5622     if (flags & 4)
5623         mask |= 0xff0000;
5624     if (flags & 8)
5625         mask |= 0xff000000;
5626     return (a & mask) | (b & ~mask);
5627 }
5628
5629 /* VFP support.  We follow the convention used for VFP instructions:
5630    Single precision routines have a "s" suffix, double precision a
5631    "d" suffix.  */
5632
5633 /* Convert host exception flags to vfp form.  */
5634 static inline int vfp_exceptbits_from_host(int host_bits)
5635 {
5636     int target_bits = 0;
5637
5638     if (host_bits & float_flag_invalid)
5639         target_bits |= 1;
5640     if (host_bits & float_flag_divbyzero)
5641         target_bits |= 2;
5642     if (host_bits & float_flag_overflow)
5643         target_bits |= 4;
5644     if (host_bits & (float_flag_underflow | float_flag_output_denormal))
5645         target_bits |= 8;
5646     if (host_bits & float_flag_inexact)
5647         target_bits |= 0x10;
5648     if (host_bits & float_flag_input_denormal)
5649         target_bits |= 0x80;
5650     return target_bits;
5651 }
5652
5653 uint32_t HELPER(vfp_get_fpscr)(CPUARMState *env)
5654 {
5655     int i;
5656     uint32_t fpscr;
5657
5658     fpscr = (env->vfp.xregs[ARM_VFP_FPSCR] & 0xffc8ffff)
5659             | (env->vfp.vec_len << 16)
5660             | (env->vfp.vec_stride << 20);
5661     i = get_float_exception_flags(&env->vfp.fp_status);
5662     i |= get_float_exception_flags(&env->vfp.standard_fp_status);
5663     fpscr |= vfp_exceptbits_from_host(i);
5664     return fpscr;
5665 }
5666
5667 uint32_t vfp_get_fpscr(CPUARMState *env)
5668 {
5669     return HELPER(vfp_get_fpscr)(env);
5670 }
5671
5672 /* Convert vfp exception flags to target form.  */
5673 static inline int vfp_exceptbits_to_host(int target_bits)
5674 {
5675     int host_bits = 0;
5676
5677     if (target_bits & 1)
5678         host_bits |= float_flag_invalid;
5679     if (target_bits & 2)
5680         host_bits |= float_flag_divbyzero;
5681     if (target_bits & 4)
5682         host_bits |= float_flag_overflow;
5683     if (target_bits & 8)
5684         host_bits |= float_flag_underflow;
5685     if (target_bits & 0x10)
5686         host_bits |= float_flag_inexact;
5687     if (target_bits & 0x80)
5688         host_bits |= float_flag_input_denormal;
5689     return host_bits;
5690 }
5691
5692 void HELPER(vfp_set_fpscr)(CPUARMState *env, uint32_t val)
5693 {
5694     int i;
5695     uint32_t changed;
5696
5697     changed = env->vfp.xregs[ARM_VFP_FPSCR];
5698     env->vfp.xregs[ARM_VFP_FPSCR] = (val & 0xffc8ffff);
5699     env->vfp.vec_len = (val >> 16) & 7;
5700     env->vfp.vec_stride = (val >> 20) & 3;
5701
5702     changed ^= val;
5703     if (changed & (3 << 22)) {
5704         i = (val >> 22) & 3;
5705         switch (i) {
5706         case FPROUNDING_TIEEVEN:
5707             i = float_round_nearest_even;
5708             break;
5709         case FPROUNDING_POSINF:
5710             i = float_round_up;
5711             break;
5712         case FPROUNDING_NEGINF:
5713             i = float_round_down;
5714             break;
5715         case FPROUNDING_ZERO:
5716             i = float_round_to_zero;
5717             break;
5718         }
5719         set_float_rounding_mode(i, &env->vfp.fp_status);
5720     }
5721     if (changed & (1 << 24)) {
5722         set_flush_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
5723         set_flush_inputs_to_zero((val & (1 << 24)) != 0, &env->vfp.fp_status);
5724     }
5725     if (changed & (1 << 25))
5726         set_default_nan_mode((val & (1 << 25)) != 0, &env->vfp.fp_status);
5727
5728     i = vfp_exceptbits_to_host(val);
5729     set_float_exception_flags(i, &env->vfp.fp_status);
5730     set_float_exception_flags(0, &env->vfp.standard_fp_status);
5731 }
5732
5733 void vfp_set_fpscr(CPUARMState *env, uint32_t val)
5734 {
5735     HELPER(vfp_set_fpscr)(env, val);
5736 }
5737
5738 #define VFP_HELPER(name, p) HELPER(glue(glue(vfp_,name),p))
5739
5740 #define VFP_BINOP(name) \
5741 float32 VFP_HELPER(name, s)(float32 a, float32 b, void *fpstp) \
5742 { \
5743     float_status *fpst = fpstp; \
5744     return float32_ ## name(a, b, fpst); \
5745 } \
5746 float64 VFP_HELPER(name, d)(float64 a, float64 b, void *fpstp) \
5747 { \
5748     float_status *fpst = fpstp; \
5749     return float64_ ## name(a, b, fpst); \
5750 }
5751 VFP_BINOP(add)
5752 VFP_BINOP(sub)
5753 VFP_BINOP(mul)
5754 VFP_BINOP(div)
5755 VFP_BINOP(min)
5756 VFP_BINOP(max)
5757 VFP_BINOP(minnum)
5758 VFP_BINOP(maxnum)
5759 #undef VFP_BINOP
5760
5761 float32 VFP_HELPER(neg, s)(float32 a)
5762 {
5763     return float32_chs(a);
5764 }
5765
5766 float64 VFP_HELPER(neg, d)(float64 a)
5767 {
5768     return float64_chs(a);
5769 }
5770
5771 float32 VFP_HELPER(abs, s)(float32 a)
5772 {
5773     return float32_abs(a);
5774 }
5775
5776 float64 VFP_HELPER(abs, d)(float64 a)
5777 {
5778     return float64_abs(a);
5779 }
5780
5781 float32 VFP_HELPER(sqrt, s)(float32 a, CPUARMState *env)
5782 {
5783     return float32_sqrt(a, &env->vfp.fp_status);
5784 }
5785
5786 float64 VFP_HELPER(sqrt, d)(float64 a, CPUARMState *env)
5787 {
5788     return float64_sqrt(a, &env->vfp.fp_status);
5789 }
5790
5791 /* XXX: check quiet/signaling case */
5792 #define DO_VFP_cmp(p, type) \
5793 void VFP_HELPER(cmp, p)(type a, type b, CPUARMState *env)  \
5794 { \
5795     uint32_t flags; \
5796     switch(type ## _compare_quiet(a, b, &env->vfp.fp_status)) { \
5797     case 0: flags = 0x6; break; \
5798     case -1: flags = 0x8; break; \
5799     case 1: flags = 0x2; break; \
5800     default: case 2: flags = 0x3; break; \
5801     } \
5802     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
5803         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
5804 } \
5805 void VFP_HELPER(cmpe, p)(type a, type b, CPUARMState *env) \
5806 { \
5807     uint32_t flags; \
5808     switch(type ## _compare(a, b, &env->vfp.fp_status)) { \
5809     case 0: flags = 0x6; break; \
5810     case -1: flags = 0x8; break; \
5811     case 1: flags = 0x2; break; \
5812     default: case 2: flags = 0x3; break; \
5813     } \
5814     env->vfp.xregs[ARM_VFP_FPSCR] = (flags << 28) \
5815         | (env->vfp.xregs[ARM_VFP_FPSCR] & 0x0fffffff); \
5816 }
5817 DO_VFP_cmp(s, float32)
5818 DO_VFP_cmp(d, float64)
5819 #undef DO_VFP_cmp
5820
5821 /* Integer to float and float to integer conversions */
5822
5823 #define CONV_ITOF(name, fsz, sign) \
5824     float##fsz HELPER(name)(uint32_t x, void *fpstp) \
5825 { \
5826     float_status *fpst = fpstp; \
5827     return sign##int32_to_##float##fsz((sign##int32_t)x, fpst); \
5828 }
5829
5830 #define CONV_FTOI(name, fsz, sign, round) \
5831 uint32_t HELPER(name)(float##fsz x, void *fpstp) \
5832 { \
5833     float_status *fpst = fpstp; \
5834     if (float##fsz##_is_any_nan(x)) { \
5835         float_raise(float_flag_invalid, fpst); \
5836         return 0; \
5837     } \
5838     return float##fsz##_to_##sign##int32##round(x, fpst); \
5839 }
5840
5841 #define FLOAT_CONVS(name, p, fsz, sign) \
5842 CONV_ITOF(vfp_##name##to##p, fsz, sign) \
5843 CONV_FTOI(vfp_to##name##p, fsz, sign, ) \
5844 CONV_FTOI(vfp_to##name##z##p, fsz, sign, _round_to_zero)
5845
5846 FLOAT_CONVS(si, s, 32, )
5847 FLOAT_CONVS(si, d, 64, )
5848 FLOAT_CONVS(ui, s, 32, u)
5849 FLOAT_CONVS(ui, d, 64, u)
5850
5851 #undef CONV_ITOF
5852 #undef CONV_FTOI
5853 #undef FLOAT_CONVS
5854
5855 /* floating point conversion */
5856 float64 VFP_HELPER(fcvtd, s)(float32 x, CPUARMState *env)
5857 {
5858     float64 r = float32_to_float64(x, &env->vfp.fp_status);
5859     /* ARM requires that S<->D conversion of any kind of NaN generates
5860      * a quiet NaN by forcing the most significant frac bit to 1.
5861      */
5862     return float64_maybe_silence_nan(r);
5863 }
5864
5865 float32 VFP_HELPER(fcvts, d)(float64 x, CPUARMState *env)
5866 {
5867     float32 r =  float64_to_float32(x, &env->vfp.fp_status);
5868     /* ARM requires that S<->D conversion of any kind of NaN generates
5869      * a quiet NaN by forcing the most significant frac bit to 1.
5870      */
5871     return float32_maybe_silence_nan(r);
5872 }
5873
5874 /* VFP3 fixed point conversion.  */
5875 #define VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype) \
5876 float##fsz HELPER(vfp_##name##to##p)(uint##isz##_t  x, uint32_t shift, \
5877                                      void *fpstp) \
5878 { \
5879     float_status *fpst = fpstp; \
5880     float##fsz tmp; \
5881     tmp = itype##_to_##float##fsz(x, fpst); \
5882     return float##fsz##_scalbn(tmp, -(int)shift, fpst); \
5883 }
5884
5885 /* Notice that we want only input-denormal exception flags from the
5886  * scalbn operation: the other possible flags (overflow+inexact if
5887  * we overflow to infinity, output-denormal) aren't correct for the
5888  * complete scale-and-convert operation.
5889  */
5890 #define VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, round) \
5891 uint##isz##_t HELPER(vfp_to##name##p##round)(float##fsz x, \
5892                                              uint32_t shift, \
5893                                              void *fpstp) \
5894 { \
5895     float_status *fpst = fpstp; \
5896     int old_exc_flags = get_float_exception_flags(fpst); \
5897     float##fsz tmp; \
5898     if (float##fsz##_is_any_nan(x)) { \
5899         float_raise(float_flag_invalid, fpst); \
5900         return 0; \
5901     } \
5902     tmp = float##fsz##_scalbn(x, shift, fpst); \
5903     old_exc_flags |= get_float_exception_flags(fpst) \
5904         & float_flag_input_denormal; \
5905     set_float_exception_flags(old_exc_flags, fpst); \
5906     return float##fsz##_to_##itype##round(tmp, fpst); \
5907 }
5908
5909 #define VFP_CONV_FIX(name, p, fsz, isz, itype)                   \
5910 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
5911 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, _round_to_zero) \
5912 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
5913
5914 #define VFP_CONV_FIX_A64(name, p, fsz, isz, itype)               \
5915 VFP_CONV_FIX_FLOAT(name, p, fsz, isz, itype)                     \
5916 VFP_CONV_FLOAT_FIX_ROUND(name, p, fsz, isz, itype, )
5917
5918 VFP_CONV_FIX(sh, d, 64, 64, int16)
5919 VFP_CONV_FIX(sl, d, 64, 64, int32)
5920 VFP_CONV_FIX_A64(sq, d, 64, 64, int64)
5921 VFP_CONV_FIX(uh, d, 64, 64, uint16)
5922 VFP_CONV_FIX(ul, d, 64, 64, uint32)
5923 VFP_CONV_FIX_A64(uq, d, 64, 64, uint64)
5924 VFP_CONV_FIX(sh, s, 32, 32, int16)
5925 VFP_CONV_FIX(sl, s, 32, 32, int32)
5926 VFP_CONV_FIX_A64(sq, s, 32, 64, int64)
5927 VFP_CONV_FIX(uh, s, 32, 32, uint16)
5928 VFP_CONV_FIX(ul, s, 32, 32, uint32)
5929 VFP_CONV_FIX_A64(uq, s, 32, 64, uint64)
5930 #undef VFP_CONV_FIX
5931 #undef VFP_CONV_FIX_FLOAT
5932 #undef VFP_CONV_FLOAT_FIX_ROUND
5933
5934 /* Set the current fp rounding mode and return the old one.
5935  * The argument is a softfloat float_round_ value.
5936  */
5937 uint32_t HELPER(set_rmode)(uint32_t rmode, CPUARMState *env)
5938 {
5939     float_status *fp_status = &env->vfp.fp_status;
5940
5941     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
5942     set_float_rounding_mode(rmode, fp_status);
5943
5944     return prev_rmode;
5945 }
5946
5947 /* Set the current fp rounding mode in the standard fp status and return
5948  * the old one. This is for NEON instructions that need to change the
5949  * rounding mode but wish to use the standard FPSCR values for everything
5950  * else. Always set the rounding mode back to the correct value after
5951  * modifying it.
5952  * The argument is a softfloat float_round_ value.
5953  */
5954 uint32_t HELPER(set_neon_rmode)(uint32_t rmode, CPUARMState *env)
5955 {
5956     float_status *fp_status = &env->vfp.standard_fp_status;
5957
5958     uint32_t prev_rmode = get_float_rounding_mode(fp_status);
5959     set_float_rounding_mode(rmode, fp_status);
5960
5961     return prev_rmode;
5962 }
5963
5964 /* Half precision conversions.  */
5965 static float32 do_fcvt_f16_to_f32(uint32_t a, CPUARMState *env, float_status *s)
5966 {
5967     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
5968     float32 r = float16_to_float32(make_float16(a), ieee, s);
5969     if (ieee) {
5970         return float32_maybe_silence_nan(r);
5971     }
5972     return r;
5973 }
5974
5975 static uint32_t do_fcvt_f32_to_f16(float32 a, CPUARMState *env, float_status *s)
5976 {
5977     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
5978     float16 r = float32_to_float16(a, ieee, s);
5979     if (ieee) {
5980         r = float16_maybe_silence_nan(r);
5981     }
5982     return float16_val(r);
5983 }
5984
5985 float32 HELPER(neon_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
5986 {
5987     return do_fcvt_f16_to_f32(a, env, &env->vfp.standard_fp_status);
5988 }
5989
5990 uint32_t HELPER(neon_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
5991 {
5992     return do_fcvt_f32_to_f16(a, env, &env->vfp.standard_fp_status);
5993 }
5994
5995 float32 HELPER(vfp_fcvt_f16_to_f32)(uint32_t a, CPUARMState *env)
5996 {
5997     return do_fcvt_f16_to_f32(a, env, &env->vfp.fp_status);
5998 }
5999
6000 uint32_t HELPER(vfp_fcvt_f32_to_f16)(float32 a, CPUARMState *env)
6001 {
6002     return do_fcvt_f32_to_f16(a, env, &env->vfp.fp_status);
6003 }
6004
6005 float64 HELPER(vfp_fcvt_f16_to_f64)(uint32_t a, CPUARMState *env)
6006 {
6007     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6008     float64 r = float16_to_float64(make_float16(a), ieee, &env->vfp.fp_status);
6009     if (ieee) {
6010         return float64_maybe_silence_nan(r);
6011     }
6012     return r;
6013 }
6014
6015 uint32_t HELPER(vfp_fcvt_f64_to_f16)(float64 a, CPUARMState *env)
6016 {
6017     int ieee = (env->vfp.xregs[ARM_VFP_FPSCR] & (1 << 26)) == 0;
6018     float16 r = float64_to_float16(a, ieee, &env->vfp.fp_status);
6019     if (ieee) {
6020         r = float16_maybe_silence_nan(r);
6021     }
6022     return float16_val(r);
6023 }
6024
6025 #define float32_two make_float32(0x40000000)
6026 #define float32_three make_float32(0x40400000)
6027 #define float32_one_point_five make_float32(0x3fc00000)
6028
6029 float32 HELPER(recps_f32)(float32 a, float32 b, CPUARMState *env)
6030 {
6031     float_status *s = &env->vfp.standard_fp_status;
6032     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
6033         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
6034         if (!(float32_is_zero(a) || float32_is_zero(b))) {
6035             float_raise(float_flag_input_denormal, s);
6036         }
6037         return float32_two;
6038     }
6039     return float32_sub(float32_two, float32_mul(a, b, s), s);
6040 }
6041
6042 float32 HELPER(rsqrts_f32)(float32 a, float32 b, CPUARMState *env)
6043 {
6044     float_status *s = &env->vfp.standard_fp_status;
6045     float32 product;
6046     if ((float32_is_infinity(a) && float32_is_zero_or_denormal(b)) ||
6047         (float32_is_infinity(b) && float32_is_zero_or_denormal(a))) {
6048         if (!(float32_is_zero(a) || float32_is_zero(b))) {
6049             float_raise(float_flag_input_denormal, s);
6050         }
6051         return float32_one_point_five;
6052     }
6053     product = float32_mul(a, b, s);
6054     return float32_div(float32_sub(float32_three, product, s), float32_two, s);
6055 }
6056
6057 /* NEON helpers.  */
6058
6059 /* Constants 256 and 512 are used in some helpers; we avoid relying on
6060  * int->float conversions at run-time.  */
6061 #define float64_256 make_float64(0x4070000000000000LL)
6062 #define float64_512 make_float64(0x4080000000000000LL)
6063 #define float32_maxnorm make_float32(0x7f7fffff)
6064 #define float64_maxnorm make_float64(0x7fefffffffffffffLL)
6065
6066 /* Reciprocal functions
6067  *
6068  * The algorithm that must be used to calculate the estimate
6069  * is specified by the ARM ARM, see FPRecipEstimate()
6070  */
6071
6072 static float64 recip_estimate(float64 a, float_status *real_fp_status)
6073 {
6074     /* These calculations mustn't set any fp exception flags,
6075      * so we use a local copy of the fp_status.
6076      */
6077     float_status dummy_status = *real_fp_status;
6078     float_status *s = &dummy_status;
6079     /* q = (int)(a * 512.0) */
6080     float64 q = float64_mul(float64_512, a, s);
6081     int64_t q_int = float64_to_int64_round_to_zero(q, s);
6082
6083     /* r = 1.0 / (((double)q + 0.5) / 512.0) */
6084     q = int64_to_float64(q_int, s);
6085     q = float64_add(q, float64_half, s);
6086     q = float64_div(q, float64_512, s);
6087     q = float64_div(float64_one, q, s);
6088
6089     /* s = (int)(256.0 * r + 0.5) */
6090     q = float64_mul(q, float64_256, s);
6091     q = float64_add(q, float64_half, s);
6092     q_int = float64_to_int64_round_to_zero(q, s);
6093
6094     /* return (double)s / 256.0 */
6095     return float64_div(int64_to_float64(q_int, s), float64_256, s);
6096 }
6097
6098 /* Common wrapper to call recip_estimate */
6099 static float64 call_recip_estimate(float64 num, int off, float_status *fpst)
6100 {
6101     uint64_t val64 = float64_val(num);
6102     uint64_t frac = extract64(val64, 0, 52);
6103     int64_t exp = extract64(val64, 52, 11);
6104     uint64_t sbit;
6105     float64 scaled, estimate;
6106
6107     /* Generate the scaled number for the estimate function */
6108     if (exp == 0) {
6109         if (extract64(frac, 51, 1) == 0) {
6110             exp = -1;
6111             frac = extract64(frac, 0, 50) << 2;
6112         } else {
6113             frac = extract64(frac, 0, 51) << 1;
6114         }
6115     }
6116
6117     /* scaled = '0' : '01111111110' : fraction<51:44> : Zeros(44); */
6118     scaled = make_float64((0x3feULL << 52)
6119                           | extract64(frac, 44, 8) << 44);
6120
6121     estimate = recip_estimate(scaled, fpst);
6122
6123     /* Build new result */
6124     val64 = float64_val(estimate);
6125     sbit = 0x8000000000000000ULL & val64;
6126     exp = off - exp;
6127     frac = extract64(val64, 0, 52);
6128
6129     if (exp == 0) {
6130         frac = 1ULL << 51 | extract64(frac, 1, 51);
6131     } else if (exp == -1) {
6132         frac = 1ULL << 50 | extract64(frac, 2, 50);
6133         exp = 0;
6134     }
6135
6136     return make_float64(sbit | (exp << 52) | frac);
6137 }
6138
6139 static bool round_to_inf(float_status *fpst, bool sign_bit)
6140 {
6141     switch (fpst->float_rounding_mode) {
6142     case float_round_nearest_even: /* Round to Nearest */
6143         return true;
6144     case float_round_up: /* Round to +Inf */
6145         return !sign_bit;
6146     case float_round_down: /* Round to -Inf */
6147         return sign_bit;
6148     case float_round_to_zero: /* Round to Zero */
6149         return false;
6150     }
6151
6152     g_assert_not_reached();
6153 }
6154
6155 float32 HELPER(recpe_f32)(float32 input, void *fpstp)
6156 {
6157     float_status *fpst = fpstp;
6158     float32 f32 = float32_squash_input_denormal(input, fpst);
6159     uint32_t f32_val = float32_val(f32);
6160     uint32_t f32_sbit = 0x80000000ULL & f32_val;
6161     int32_t f32_exp = extract32(f32_val, 23, 8);
6162     uint32_t f32_frac = extract32(f32_val, 0, 23);
6163     float64 f64, r64;
6164     uint64_t r64_val;
6165     int64_t r64_exp;
6166     uint64_t r64_frac;
6167
6168     if (float32_is_any_nan(f32)) {
6169         float32 nan = f32;
6170         if (float32_is_signaling_nan(f32)) {
6171             float_raise(float_flag_invalid, fpst);
6172             nan = float32_maybe_silence_nan(f32);
6173         }
6174         if (fpst->default_nan_mode) {
6175             nan =  float32_default_nan;
6176         }
6177         return nan;
6178     } else if (float32_is_infinity(f32)) {
6179         return float32_set_sign(float32_zero, float32_is_neg(f32));
6180     } else if (float32_is_zero(f32)) {
6181         float_raise(float_flag_divbyzero, fpst);
6182         return float32_set_sign(float32_infinity, float32_is_neg(f32));
6183     } else if ((f32_val & ~(1ULL << 31)) < (1ULL << 21)) {
6184         /* Abs(value) < 2.0^-128 */
6185         float_raise(float_flag_overflow | float_flag_inexact, fpst);
6186         if (round_to_inf(fpst, f32_sbit)) {
6187             return float32_set_sign(float32_infinity, float32_is_neg(f32));
6188         } else {
6189             return float32_set_sign(float32_maxnorm, float32_is_neg(f32));
6190         }
6191     } else if (f32_exp >= 253 && fpst->flush_to_zero) {
6192         float_raise(float_flag_underflow, fpst);
6193         return float32_set_sign(float32_zero, float32_is_neg(f32));
6194     }
6195
6196
6197     f64 = make_float64(((int64_t)(f32_exp) << 52) | (int64_t)(f32_frac) << 29);
6198     r64 = call_recip_estimate(f64, 253, fpst);
6199     r64_val = float64_val(r64);
6200     r64_exp = extract64(r64_val, 52, 11);
6201     r64_frac = extract64(r64_val, 0, 52);
6202
6203     /* result = sign : result_exp<7:0> : fraction<51:29>; */
6204     return make_float32(f32_sbit |
6205                         (r64_exp & 0xff) << 23 |
6206                         extract64(r64_frac, 29, 24));
6207 }
6208
6209 float64 HELPER(recpe_f64)(float64 input, void *fpstp)
6210 {
6211     float_status *fpst = fpstp;
6212     float64 f64 = float64_squash_input_denormal(input, fpst);
6213     uint64_t f64_val = float64_val(f64);
6214     uint64_t f64_sbit = 0x8000000000000000ULL & f64_val;
6215     int64_t f64_exp = extract64(f64_val, 52, 11);
6216     float64 r64;
6217     uint64_t r64_val;
6218     int64_t r64_exp;
6219     uint64_t r64_frac;
6220
6221     /* Deal with any special cases */
6222     if (float64_is_any_nan(f64)) {
6223         float64 nan = f64;
6224         if (float64_is_signaling_nan(f64)) {
6225             float_raise(float_flag_invalid, fpst);
6226             nan = float64_maybe_silence_nan(f64);
6227         }
6228         if (fpst->default_nan_mode) {
6229             nan =  float64_default_nan;
6230         }
6231         return nan;
6232     } else if (float64_is_infinity(f64)) {
6233         return float64_set_sign(float64_zero, float64_is_neg(f64));
6234     } else if (float64_is_zero(f64)) {
6235         float_raise(float_flag_divbyzero, fpst);
6236         return float64_set_sign(float64_infinity, float64_is_neg(f64));
6237     } else if ((f64_val & ~(1ULL << 63)) < (1ULL << 50)) {
6238         /* Abs(value) < 2.0^-1024 */
6239         float_raise(float_flag_overflow | float_flag_inexact, fpst);
6240         if (round_to_inf(fpst, f64_sbit)) {
6241             return float64_set_sign(float64_infinity, float64_is_neg(f64));
6242         } else {
6243             return float64_set_sign(float64_maxnorm, float64_is_neg(f64));
6244         }
6245     } else if (f64_exp >= 1023 && fpst->flush_to_zero) {
6246         float_raise(float_flag_underflow, fpst);
6247         return float64_set_sign(float64_zero, float64_is_neg(f64));
6248     }
6249
6250     r64 = call_recip_estimate(f64, 2045, fpst);
6251     r64_val = float64_val(r64);
6252     r64_exp = extract64(r64_val, 52, 11);
6253     r64_frac = extract64(r64_val, 0, 52);
6254
6255     /* result = sign : result_exp<10:0> : fraction<51:0> */
6256     return make_float64(f64_sbit |
6257                         ((r64_exp & 0x7ff) << 52) |
6258                         r64_frac);
6259 }
6260
6261 /* The algorithm that must be used to calculate the estimate
6262  * is specified by the ARM ARM.
6263  */
6264 static float64 recip_sqrt_estimate(float64 a, float_status *real_fp_status)
6265 {
6266     /* These calculations mustn't set any fp exception flags,
6267      * so we use a local copy of the fp_status.
6268      */
6269     float_status dummy_status = *real_fp_status;
6270     float_status *s = &dummy_status;
6271     float64 q;
6272     int64_t q_int;
6273
6274     if (float64_lt(a, float64_half, s)) {
6275         /* range 0.25 <= a < 0.5 */
6276
6277         /* a in units of 1/512 rounded down */
6278         /* q0 = (int)(a * 512.0);  */
6279         q = float64_mul(float64_512, a, s);
6280         q_int = float64_to_int64_round_to_zero(q, s);
6281
6282         /* reciprocal root r */
6283         /* r = 1.0 / sqrt(((double)q0 + 0.5) / 512.0);  */
6284         q = int64_to_float64(q_int, s);
6285         q = float64_add(q, float64_half, s);
6286         q = float64_div(q, float64_512, s);
6287         q = float64_sqrt(q, s);
6288         q = float64_div(float64_one, q, s);
6289     } else {
6290         /* range 0.5 <= a < 1.0 */
6291
6292         /* a in units of 1/256 rounded down */
6293         /* q1 = (int)(a * 256.0); */
6294         q = float64_mul(float64_256, a, s);
6295         int64_t q_int = float64_to_int64_round_to_zero(q, s);
6296
6297         /* reciprocal root r */
6298         /* r = 1.0 /sqrt(((double)q1 + 0.5) / 256); */
6299         q = int64_to_float64(q_int, s);
6300         q = float64_add(q, float64_half, s);
6301         q = float64_div(q, float64_256, s);
6302         q = float64_sqrt(q, s);
6303         q = float64_div(float64_one, q, s);
6304     }
6305     /* r in units of 1/256 rounded to nearest */
6306     /* s = (int)(256.0 * r + 0.5); */
6307
6308     q = float64_mul(q, float64_256,s );
6309     q = float64_add(q, float64_half, s);
6310     q_int = float64_to_int64_round_to_zero(q, s);
6311
6312     /* return (double)s / 256.0;*/
6313     return float64_div(int64_to_float64(q_int, s), float64_256, s);
6314 }
6315
6316 float32 HELPER(rsqrte_f32)(float32 input, void *fpstp)
6317 {
6318     float_status *s = fpstp;
6319     float32 f32 = float32_squash_input_denormal(input, s);
6320     uint32_t val = float32_val(f32);
6321     uint32_t f32_sbit = 0x80000000 & val;
6322     int32_t f32_exp = extract32(val, 23, 8);
6323     uint32_t f32_frac = extract32(val, 0, 23);
6324     uint64_t f64_frac;
6325     uint64_t val64;
6326     int result_exp;
6327     float64 f64;
6328
6329     if (float32_is_any_nan(f32)) {
6330         float32 nan = f32;
6331         if (float32_is_signaling_nan(f32)) {
6332             float_raise(float_flag_invalid, s);
6333             nan = float32_maybe_silence_nan(f32);
6334         }
6335         if (s->default_nan_mode) {
6336             nan =  float32_default_nan;
6337         }
6338         return nan;
6339     } else if (float32_is_zero(f32)) {
6340         float_raise(float_flag_divbyzero, s);
6341         return float32_set_sign(float32_infinity, float32_is_neg(f32));
6342     } else if (float32_is_neg(f32)) {
6343         float_raise(float_flag_invalid, s);
6344         return float32_default_nan;
6345     } else if (float32_is_infinity(f32)) {
6346         return float32_zero;
6347     }
6348
6349     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
6350      * preserving the parity of the exponent.  */
6351
6352     f64_frac = ((uint64_t) f32_frac) << 29;
6353     if (f32_exp == 0) {
6354         while (extract64(f64_frac, 51, 1) == 0) {
6355             f64_frac = f64_frac << 1;
6356             f32_exp = f32_exp-1;
6357         }
6358         f64_frac = extract64(f64_frac, 0, 51) << 1;
6359     }
6360
6361     if (extract64(f32_exp, 0, 1) == 0) {
6362         f64 = make_float64(((uint64_t) f32_sbit) << 32
6363                            | (0x3feULL << 52)
6364                            | f64_frac);
6365     } else {
6366         f64 = make_float64(((uint64_t) f32_sbit) << 32
6367                            | (0x3fdULL << 52)
6368                            | f64_frac);
6369     }
6370
6371     result_exp = (380 - f32_exp) / 2;
6372
6373     f64 = recip_sqrt_estimate(f64, s);
6374
6375     val64 = float64_val(f64);
6376
6377     val = ((result_exp & 0xff) << 23)
6378         | ((val64 >> 29)  & 0x7fffff);
6379     return make_float32(val);
6380 }
6381
6382 float64 HELPER(rsqrte_f64)(float64 input, void *fpstp)
6383 {
6384     float_status *s = fpstp;
6385     float64 f64 = float64_squash_input_denormal(input, s);
6386     uint64_t val = float64_val(f64);
6387     uint64_t f64_sbit = 0x8000000000000000ULL & val;
6388     int64_t f64_exp = extract64(val, 52, 11);
6389     uint64_t f64_frac = extract64(val, 0, 52);
6390     int64_t result_exp;
6391     uint64_t result_frac;
6392
6393     if (float64_is_any_nan(f64)) {
6394         float64 nan = f64;
6395         if (float64_is_signaling_nan(f64)) {
6396             float_raise(float_flag_invalid, s);
6397             nan = float64_maybe_silence_nan(f64);
6398         }
6399         if (s->default_nan_mode) {
6400             nan =  float64_default_nan;
6401         }
6402         return nan;
6403     } else if (float64_is_zero(f64)) {
6404         float_raise(float_flag_divbyzero, s);
6405         return float64_set_sign(float64_infinity, float64_is_neg(f64));
6406     } else if (float64_is_neg(f64)) {
6407         float_raise(float_flag_invalid, s);
6408         return float64_default_nan;
6409     } else if (float64_is_infinity(f64)) {
6410         return float64_zero;
6411     }
6412
6413     /* Scale and normalize to a double-precision value between 0.25 and 1.0,
6414      * preserving the parity of the exponent.  */
6415
6416     if (f64_exp == 0) {
6417         while (extract64(f64_frac, 51, 1) == 0) {
6418             f64_frac = f64_frac << 1;
6419             f64_exp = f64_exp - 1;
6420         }
6421         f64_frac = extract64(f64_frac, 0, 51) << 1;
6422     }
6423
6424     if (extract64(f64_exp, 0, 1) == 0) {
6425         f64 = make_float64(f64_sbit
6426                            | (0x3feULL << 52)
6427                            | f64_frac);
6428     } else {
6429         f64 = make_float64(f64_sbit
6430                            | (0x3fdULL << 52)
6431                            | f64_frac);
6432     }
6433
6434     result_exp = (3068 - f64_exp) / 2;
6435
6436     f64 = recip_sqrt_estimate(f64, s);
6437
6438     result_frac = extract64(float64_val(f64), 0, 52);
6439
6440     return make_float64(f64_sbit |
6441                         ((result_exp & 0x7ff) << 52) |
6442                         result_frac);
6443 }
6444
6445 uint32_t HELPER(recpe_u32)(uint32_t a, void *fpstp)
6446 {
6447     float_status *s = fpstp;
6448     float64 f64;
6449
6450     if ((a & 0x80000000) == 0) {
6451         return 0xffffffff;
6452     }
6453
6454     f64 = make_float64((0x3feULL << 52)
6455                        | ((int64_t)(a & 0x7fffffff) << 21));
6456
6457     f64 = recip_estimate(f64, s);
6458
6459     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
6460 }
6461
6462 uint32_t HELPER(rsqrte_u32)(uint32_t a, void *fpstp)
6463 {
6464     float_status *fpst = fpstp;
6465     float64 f64;
6466
6467     if ((a & 0xc0000000) == 0) {
6468         return 0xffffffff;
6469     }
6470
6471     if (a & 0x80000000) {
6472         f64 = make_float64((0x3feULL << 52)
6473                            | ((uint64_t)(a & 0x7fffffff) << 21));
6474     } else { /* bits 31-30 == '01' */
6475         f64 = make_float64((0x3fdULL << 52)
6476                            | ((uint64_t)(a & 0x3fffffff) << 22));
6477     }
6478
6479     f64 = recip_sqrt_estimate(f64, fpst);
6480
6481     return 0x80000000 | ((float64_val(f64) >> 21) & 0x7fffffff);
6482 }
6483
6484 /* VFPv4 fused multiply-accumulate */
6485 float32 VFP_HELPER(muladd, s)(float32 a, float32 b, float32 c, void *fpstp)
6486 {
6487     float_status *fpst = fpstp;
6488     return float32_muladd(a, b, c, 0, fpst);
6489 }
6490
6491 float64 VFP_HELPER(muladd, d)(float64 a, float64 b, float64 c, void *fpstp)
6492 {
6493     float_status *fpst = fpstp;
6494     return float64_muladd(a, b, c, 0, fpst);
6495 }
6496
6497 /* ARMv8 round to integral */
6498 float32 HELPER(rints_exact)(float32 x, void *fp_status)
6499 {
6500     return float32_round_to_int(x, fp_status);
6501 }
6502
6503 float64 HELPER(rintd_exact)(float64 x, void *fp_status)
6504 {
6505     return float64_round_to_int(x, fp_status);
6506 }
6507
6508 float32 HELPER(rints)(float32 x, void *fp_status)
6509 {
6510     int old_flags = get_float_exception_flags(fp_status), new_flags;
6511     float32 ret;
6512
6513     ret = float32_round_to_int(x, fp_status);
6514
6515     /* Suppress any inexact exceptions the conversion produced */
6516     if (!(old_flags & float_flag_inexact)) {
6517         new_flags = get_float_exception_flags(fp_status);
6518         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
6519     }
6520
6521     return ret;
6522 }
6523
6524 float64 HELPER(rintd)(float64 x, void *fp_status)
6525 {
6526     int old_flags = get_float_exception_flags(fp_status), new_flags;
6527     float64 ret;
6528
6529     ret = float64_round_to_int(x, fp_status);
6530
6531     new_flags = get_float_exception_flags(fp_status);
6532
6533     /* Suppress any inexact exceptions the conversion produced */
6534     if (!(old_flags & float_flag_inexact)) {
6535         new_flags = get_float_exception_flags(fp_status);
6536         set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status);
6537     }
6538
6539     return ret;
6540 }
6541
6542 /* Convert ARM rounding mode to softfloat */
6543 int arm_rmode_to_sf(int rmode)
6544 {
6545     switch (rmode) {
6546     case FPROUNDING_TIEAWAY:
6547         rmode = float_round_ties_away;
6548         break;
6549     case FPROUNDING_ODD:
6550         /* FIXME: add support for TIEAWAY and ODD */
6551         qemu_log_mask(LOG_UNIMP, "arm: unimplemented rounding mode: %d\n",
6552                       rmode);
6553     case FPROUNDING_TIEEVEN:
6554     default:
6555         rmode = float_round_nearest_even;
6556         break;
6557     case FPROUNDING_POSINF:
6558         rmode = float_round_up;
6559         break;
6560     case FPROUNDING_NEGINF:
6561         rmode = float_round_down;
6562         break;
6563     case FPROUNDING_ZERO:
6564         rmode = float_round_to_zero;
6565         break;
6566     }
6567     return rmode;
6568 }
6569
6570 /* CRC helpers.
6571  * The upper bytes of val (above the number specified by 'bytes') must have
6572  * been zeroed out by the caller.
6573  */
6574 uint32_t HELPER(crc32)(uint32_t acc, uint32_t val, uint32_t bytes)
6575 {
6576     uint8_t buf[4];
6577
6578     stl_le_p(buf, val);
6579
6580     /* zlib crc32 converts the accumulator and output to one's complement.  */
6581     return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff;
6582 }
6583
6584 uint32_t HELPER(crc32c)(uint32_t acc, uint32_t val, uint32_t bytes)
6585 {
6586     uint8_t buf[4];
6587
6588     stl_le_p(buf, val);
6589
6590     /* Linux crc32c converts the output to one's complement.  */
6591     return crc32c(acc, buf, bytes) ^ 0xffffffff;
6592 }