target/arm/machine.c

   1 #include "qemu/osdep.h"
   2 #include "cpu.h"
   3 #include "qemu/error-report.h"
   4 #include "sysemu/kvm.h"
   5 #include "sysemu/tcg.h"
   6 #include "kvm_arm.h"
   7 #include "internals.h"
   8 #include "migration/cpu.h"
   9
  10 static bool vfp_needed(void *opaque)
  11 {
  12     ARMCPU *cpu = opaque;
  13
  14     return (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)
  15             ? cpu_isar_feature(aa64_fp_simd, cpu)
  16             : cpu_isar_feature(aa32_vfp_simd, cpu));
  17 }
  18
  19 static int get_fpscr(QEMUFile *f, void *opaque, size_t size,
  20                      const VMStateField *field)
  21 {
  22     ARMCPU *cpu = opaque;
  23     CPUARMState *env = &cpu->env;
  24     uint32_t val = qemu_get_be32(f);
  25
  26     vfp_set_fpscr(env, val);
  27     return 0;
  28 }
  29
  30 static int put_fpscr(QEMUFile *f, void *opaque, size_t size,
  31                      const VMStateField *field, JSONWriter *vmdesc)
  32 {
  33     ARMCPU *cpu = opaque;
  34     CPUARMState *env = &cpu->env;
  35
  36     qemu_put_be32(f, vfp_get_fpscr(env));
  37     return 0;
  38 }
  39
  40 static const VMStateInfo vmstate_fpscr = {
  41     .name = "fpscr",
  42     .get = get_fpscr,
  43     .put = put_fpscr,
  44 };
  45
  46 static const VMStateDescription vmstate_vfp = {
  47     .name = "cpu/vfp",
  48     .version_id = 3,
  49     .minimum_version_id = 3,
  50     .needed = vfp_needed,
  51     .fields = (VMStateField[]) {
  52         /* For compatibility, store Qn out of Zn here.  */
  53         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[0].d, ARMCPU, 0, 2),
  54         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[1].d, ARMCPU, 0, 2),
  55         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[2].d, ARMCPU, 0, 2),
  56         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[3].d, ARMCPU, 0, 2),
  57         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[4].d, ARMCPU, 0, 2),
  58         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[5].d, ARMCPU, 0, 2),
  59         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[6].d, ARMCPU, 0, 2),
  60         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[7].d, ARMCPU, 0, 2),
  61         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[8].d, ARMCPU, 0, 2),
  62         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[9].d, ARMCPU, 0, 2),
  63         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[10].d, ARMCPU, 0, 2),
  64         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[11].d, ARMCPU, 0, 2),
  65         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[12].d, ARMCPU, 0, 2),
  66         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[13].d, ARMCPU, 0, 2),
  67         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[14].d, ARMCPU, 0, 2),
  68         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[15].d, ARMCPU, 0, 2),
  69         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[16].d, ARMCPU, 0, 2),
  70         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[17].d, ARMCPU, 0, 2),
  71         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[18].d, ARMCPU, 0, 2),
  72         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[19].d, ARMCPU, 0, 2),
  73         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[20].d, ARMCPU, 0, 2),
  74         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[21].d, ARMCPU, 0, 2),
  75         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[22].d, ARMCPU, 0, 2),
  76         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[23].d, ARMCPU, 0, 2),
  77         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[24].d, ARMCPU, 0, 2),
  78         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[25].d, ARMCPU, 0, 2),
  79         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[26].d, ARMCPU, 0, 2),
  80         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[27].d, ARMCPU, 0, 2),
  81         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[28].d, ARMCPU, 0, 2),
  82         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[29].d, ARMCPU, 0, 2),
  83         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[30].d, ARMCPU, 0, 2),
  84         VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[31].d, ARMCPU, 0, 2),
  85
  86         /* The xregs array is a little awkward because element 1 (FPSCR)
  87          * requires a specific accessor, so we have to split it up in
  88          * the vmstate:
  89          */
  90         VMSTATE_UINT32(env.vfp.xregs[0], ARMCPU),
  91         VMSTATE_UINT32_SUB_ARRAY(env.vfp.xregs, ARMCPU, 2, 14),
  92         {
  93             .name = "fpscr",
  94             .version_id = 0,
  95             .size = sizeof(uint32_t),
  96             .info = &vmstate_fpscr,
  97             .flags = VMS_SINGLE,
  98             .offset = 0,
  99         },
 100         VMSTATE_END_OF_LIST()
 101     }
 102 };
 103
 104 static bool iwmmxt_needed(void *opaque)
 105 {
 106     ARMCPU *cpu = opaque;
 107     CPUARMState *env = &cpu->env;
 108
 109     return arm_feature(env, ARM_FEATURE_IWMMXT);
 110 }
 111
 112 static const VMStateDescription vmstate_iwmmxt = {
 113     .name = "cpu/iwmmxt",
 114     .version_id = 1,
 115     .minimum_version_id = 1,
 116     .needed = iwmmxt_needed,
 117     .fields = (VMStateField[]) {
 118         VMSTATE_UINT64_ARRAY(env.iwmmxt.regs, ARMCPU, 16),
 119         VMSTATE_UINT32_ARRAY(env.iwmmxt.cregs, ARMCPU, 16),
 120         VMSTATE_END_OF_LIST()
 121     }
 122 };
 123
 124 #ifdef TARGET_AARCH64
 125 /* The expression ARM_MAX_VQ - 2 is 0 for pure AArch32 build,
 126  * and ARMPredicateReg is actively empty.  This triggers errors
 127  * in the expansion of the VMSTATE macros.
 128  */
 129
 130 static bool sve_needed(void *opaque)
 131 {
 132     ARMCPU *cpu = opaque;
 133
 134     return cpu_isar_feature(aa64_sve, cpu);
 135 }
 136
 137 /* The first two words of each Zreg is stored in VFP state.  */
 138 static const VMStateDescription vmstate_zreg_hi_reg = {
 139     .name = "cpu/sve/zreg_hi",
 140     .version_id = 1,
 141     .minimum_version_id = 1,
 142     .fields = (VMStateField[]) {
 143         VMSTATE_UINT64_SUB_ARRAY(d, ARMVectorReg, 2, ARM_MAX_VQ - 2),
 144         VMSTATE_END_OF_LIST()
 145     }
 146 };
 147
 148 static const VMStateDescription vmstate_preg_reg = {
 149     .name = "cpu/sve/preg",
 150     .version_id = 1,
 151     .minimum_version_id = 1,
 152     .fields = (VMStateField[]) {
 153         VMSTATE_UINT64_ARRAY(p, ARMPredicateReg, 2 * ARM_MAX_VQ / 8),
 154         VMSTATE_END_OF_LIST()
 155     }
 156 };
 157
 158 static const VMStateDescription vmstate_sve = {
 159     .name = "cpu/sve",
 160     .version_id = 1,
 161     .minimum_version_id = 1,
 162     .needed = sve_needed,
 163     .fields = (VMStateField[]) {
 164         VMSTATE_STRUCT_ARRAY(env.vfp.zregs, ARMCPU, 32, 0,
 165                              vmstate_zreg_hi_reg, ARMVectorReg),
 166         VMSTATE_STRUCT_ARRAY(env.vfp.pregs, ARMCPU, 17, 0,
 167                              vmstate_preg_reg, ARMPredicateReg),
 168         VMSTATE_END_OF_LIST()
 169     }
 170 };
 171
 172 static const VMStateDescription vmstate_vreg = {
 173     .name = "vreg",
 174     .version_id = 1,
 175     .minimum_version_id = 1,
 176     .fields = (VMStateField[]) {
 177         VMSTATE_UINT64_ARRAY(d, ARMVectorReg, ARM_MAX_VQ * 2),
 178         VMSTATE_END_OF_LIST()
 179     }
 180 };
 181
 182 static bool za_needed(void *opaque)
 183 {
 184     ARMCPU *cpu = opaque;
 185
 186     /*
 187      * When ZA storage is disabled, its contents are discarded.
 188      * It will be zeroed when ZA storage is re-enabled.
 189      */
 190     return FIELD_EX64(cpu->env.svcr, SVCR, ZA);
 191 }
 192
 193 static const VMStateDescription vmstate_za = {
 194     .name = "cpu/sme",
 195     .version_id = 1,
 196     .minimum_version_id = 1,
 197     .needed = za_needed,
 198     .fields = (VMStateField[]) {
 199         VMSTATE_STRUCT_ARRAY(env.zarray, ARMCPU, ARM_MAX_VQ * 16, 0,
 200                              vmstate_vreg, ARMVectorReg),
 201         VMSTATE_END_OF_LIST()
 202     }
 203 };
 204 #endif /* AARCH64 */
 205
 206 static bool serror_needed(void *opaque)
 207 {
 208     ARMCPU *cpu = opaque;
 209     CPUARMState *env = &cpu->env;
 210
 211     return env->serror.pending != 0;
 212 }
 213
 214 static const VMStateDescription vmstate_serror = {
 215     .name = "cpu/serror",
 216     .version_id = 1,
 217     .minimum_version_id = 1,
 218     .needed = serror_needed,
 219     .fields = (VMStateField[]) {
 220         VMSTATE_UINT8(env.serror.pending, ARMCPU),
 221         VMSTATE_UINT8(env.serror.has_esr, ARMCPU),
 222         VMSTATE_UINT64(env.serror.esr, ARMCPU),
 223         VMSTATE_END_OF_LIST()
 224     }
 225 };
 226
 227 static bool irq_line_state_needed(void *opaque)
 228 {
 229     return true;
 230 }
 231
 232 static const VMStateDescription vmstate_irq_line_state = {
 233     .name = "cpu/irq-line-state",
 234     .version_id = 1,
 235     .minimum_version_id = 1,
 236     .needed = irq_line_state_needed,
 237     .fields = (VMStateField[]) {
 238         VMSTATE_UINT32(env.irq_line_state, ARMCPU),
 239         VMSTATE_END_OF_LIST()
 240     }
 241 };
 242
 243 static bool m_needed(void *opaque)
 244 {
 245     ARMCPU *cpu = opaque;
 246     CPUARMState *env = &cpu->env;
 247
 248     return arm_feature(env, ARM_FEATURE_M);
 249 }
 250
 251 static const VMStateDescription vmstate_m_faultmask_primask = {
 252     .name = "cpu/m/faultmask-primask",
 253     .version_id = 1,
 254     .minimum_version_id = 1,
 255     .needed = m_needed,
 256     .fields = (VMStateField[]) {
 257         VMSTATE_UINT32(env.v7m.faultmask[M_REG_NS], ARMCPU),
 258         VMSTATE_UINT32(env.v7m.primask[M_REG_NS], ARMCPU),
 259         VMSTATE_END_OF_LIST()
 260     }
 261 };
 262
 263 /* CSSELR is in a subsection because we didn't implement it previously.
 264  * Migration from an old implementation will leave it at zero, which
 265  * is OK since the only CPUs in the old implementation make the
 266  * register RAZ/WI.
 267  * Since there was no version of QEMU which implemented the CSSELR for
 268  * just non-secure, we transfer both banks here rather than putting
 269  * the secure banked version in the m-security subsection.
 270  */
 271 static bool csselr_vmstate_validate(void *opaque, int version_id)
 272 {
 273     ARMCPU *cpu = opaque;
 274
 275     return cpu->env.v7m.csselr[M_REG_NS] <= R_V7M_CSSELR_INDEX_MASK
 276         && cpu->env.v7m.csselr[M_REG_S] <= R_V7M_CSSELR_INDEX_MASK;
 277 }
 278
 279 static bool m_csselr_needed(void *opaque)
 280 {
 281     ARMCPU *cpu = opaque;
 282
 283     return !arm_v7m_csselr_razwi(cpu);
 284 }
 285
 286 static const VMStateDescription vmstate_m_csselr = {
 287     .name = "cpu/m/csselr",
 288     .version_id = 1,
 289     .minimum_version_id = 1,
 290     .needed = m_csselr_needed,
 291     .fields = (VMStateField[]) {
 292         VMSTATE_UINT32_ARRAY(env.v7m.csselr, ARMCPU, M_REG_NUM_BANKS),
 293         VMSTATE_VALIDATE("CSSELR is valid", csselr_vmstate_validate),
 294         VMSTATE_END_OF_LIST()
 295     }
 296 };
 297
 298 static const VMStateDescription vmstate_m_scr = {
 299     .name = "cpu/m/scr",
 300     .version_id = 1,
 301     .minimum_version_id = 1,
 302     .needed = m_needed,
 303     .fields = (VMStateField[]) {
 304         VMSTATE_UINT32(env.v7m.scr[M_REG_NS], ARMCPU),
 305         VMSTATE_END_OF_LIST()
 306     }
 307 };
 308
 309 static const VMStateDescription vmstate_m_other_sp = {
 310     .name = "cpu/m/other-sp",
 311     .version_id = 1,
 312     .minimum_version_id = 1,
 313     .needed = m_needed,
 314     .fields = (VMStateField[]) {
 315         VMSTATE_UINT32(env.v7m.other_sp, ARMCPU),
 316         VMSTATE_END_OF_LIST()
 317     }
 318 };
 319
 320 static bool m_v8m_needed(void *opaque)
 321 {
 322     ARMCPU *cpu = opaque;
 323     CPUARMState *env = &cpu->env;
 324
 325     return arm_feature(env, ARM_FEATURE_M) && arm_feature(env, ARM_FEATURE_V8);
 326 }
 327
 328 static const VMStateDescription vmstate_m_v8m = {
 329     .name = "cpu/m/v8m",
 330     .version_id = 1,
 331     .minimum_version_id = 1,
 332     .needed = m_v8m_needed,
 333     .fields = (VMStateField[]) {
 334         VMSTATE_UINT32_ARRAY(env.v7m.msplim, ARMCPU, M_REG_NUM_BANKS),
 335         VMSTATE_UINT32_ARRAY(env.v7m.psplim, ARMCPU, M_REG_NUM_BANKS),
 336         VMSTATE_END_OF_LIST()
 337     }
 338 };
 339
 340 static const VMStateDescription vmstate_m_fp = {
 341     .name = "cpu/m/fp",
 342     .version_id = 1,
 343     .minimum_version_id = 1,
 344     .needed = vfp_needed,
 345     .fields = (VMStateField[]) {
 346         VMSTATE_UINT32_ARRAY(env.v7m.fpcar, ARMCPU, M_REG_NUM_BANKS),
 347         VMSTATE_UINT32_ARRAY(env.v7m.fpccr, ARMCPU, M_REG_NUM_BANKS),
 348         VMSTATE_UINT32_ARRAY(env.v7m.fpdscr, ARMCPU, M_REG_NUM_BANKS),
 349         VMSTATE_UINT32_ARRAY(env.v7m.cpacr, ARMCPU, M_REG_NUM_BANKS),
 350         VMSTATE_UINT32(env.v7m.nsacr, ARMCPU),
 351         VMSTATE_END_OF_LIST()
 352     }
 353 };
 354
 355 static bool mve_needed(void *opaque)
 356 {
 357     ARMCPU *cpu = opaque;
 358
 359     return cpu_isar_feature(aa32_mve, cpu);
 360 }
 361
 362 static const VMStateDescription vmstate_m_mve = {
 363     .name = "cpu/m/mve",
 364     .version_id = 1,
 365     .minimum_version_id = 1,
 366     .needed = mve_needed,
 367     .fields = (VMStateField[]) {
 368         VMSTATE_UINT32(env.v7m.vpr, ARMCPU),
 369         VMSTATE_UINT32(env.v7m.ltpsize, ARMCPU),
 370         VMSTATE_END_OF_LIST()
 371     },
 372 };
 373
 374 static const VMStateDescription vmstate_m = {
 375     .name = "cpu/m",
 376     .version_id = 4,
 377     .minimum_version_id = 4,
 378     .needed = m_needed,
 379     .fields = (VMStateField[]) {
 380         VMSTATE_UINT32(env.v7m.vecbase[M_REG_NS], ARMCPU),
 381         VMSTATE_UINT32(env.v7m.basepri[M_REG_NS], ARMCPU),
 382         VMSTATE_UINT32(env.v7m.control[M_REG_NS], ARMCPU),
 383         VMSTATE_UINT32(env.v7m.ccr[M_REG_NS], ARMCPU),
 384         VMSTATE_UINT32(env.v7m.cfsr[M_REG_NS], ARMCPU),
 385         VMSTATE_UINT32(env.v7m.hfsr, ARMCPU),
 386         VMSTATE_UINT32(env.v7m.dfsr, ARMCPU),
 387         VMSTATE_UINT32(env.v7m.mmfar[M_REG_NS], ARMCPU),
 388         VMSTATE_UINT32(env.v7m.bfar, ARMCPU),
 389         VMSTATE_UINT32(env.v7m.mpu_ctrl[M_REG_NS], ARMCPU),
 390         VMSTATE_INT32(env.v7m.exception, ARMCPU),
 391         VMSTATE_END_OF_LIST()
 392     },
 393     .subsections = (const VMStateDescription*[]) {
 394         &vmstate_m_faultmask_primask,
 395         &vmstate_m_csselr,
 396         &vmstate_m_scr,
 397         &vmstate_m_other_sp,
 398         &vmstate_m_v8m,
 399         &vmstate_m_fp,
 400         &vmstate_m_mve,
 401         NULL
 402     }
 403 };
 404
 405 static bool thumb2ee_needed(void *opaque)
 406 {
 407     ARMCPU *cpu = opaque;
 408     CPUARMState *env = &cpu->env;
 409
 410     return arm_feature(env, ARM_FEATURE_THUMB2EE);
 411 }
 412
 413 static const VMStateDescription vmstate_thumb2ee = {
 414     .name = "cpu/thumb2ee",
 415     .version_id = 1,
 416     .minimum_version_id = 1,
 417     .needed = thumb2ee_needed,
 418     .fields = (VMStateField[]) {
 419         VMSTATE_UINT32(env.teecr, ARMCPU),
 420         VMSTATE_UINT32(env.teehbr, ARMCPU),
 421         VMSTATE_END_OF_LIST()
 422     }
 423 };
 424
 425 static bool pmsav7_needed(void *opaque)
 426 {
 427     ARMCPU *cpu = opaque;
 428     CPUARMState *env = &cpu->env;
 429
 430     return arm_feature(env, ARM_FEATURE_PMSA) &&
 431            arm_feature(env, ARM_FEATURE_V7) &&
 432            !arm_feature(env, ARM_FEATURE_V8);
 433 }
 434
 435 static bool pmsav7_rgnr_vmstate_validate(void *opaque, int version_id)
 436 {
 437     ARMCPU *cpu = opaque;
 438
 439     return cpu->env.pmsav7.rnr[M_REG_NS] < cpu->pmsav7_dregion;
 440 }
 441
 442 static const VMStateDescription vmstate_pmsav7 = {
 443     .name = "cpu/pmsav7",
 444     .version_id = 1,
 445     .minimum_version_id = 1,
 446     .needed = pmsav7_needed,
 447     .fields = (VMStateField[]) {
 448         VMSTATE_VARRAY_UINT32(env.pmsav7.drbar, ARMCPU, pmsav7_dregion, 0,
 449                               vmstate_info_uint32, uint32_t),
 450         VMSTATE_VARRAY_UINT32(env.pmsav7.drsr, ARMCPU, pmsav7_dregion, 0,
 451                               vmstate_info_uint32, uint32_t),
 452         VMSTATE_VARRAY_UINT32(env.pmsav7.dracr, ARMCPU, pmsav7_dregion, 0,
 453                               vmstate_info_uint32, uint32_t),
 454         VMSTATE_VALIDATE("rgnr is valid", pmsav7_rgnr_vmstate_validate),
 455         VMSTATE_END_OF_LIST()
 456     }
 457 };
 458
 459 static bool pmsav7_rnr_needed(void *opaque)
 460 {
 461     ARMCPU *cpu = opaque;
 462     CPUARMState *env = &cpu->env;
 463
 464     /* For R profile cores pmsav7.rnr is migrated via the cpreg
 465      * "RGNR" definition in helper.h. For M profile we have to
 466      * migrate it separately.
 467      */
 468     return arm_feature(env, ARM_FEATURE_M);
 469 }
 470
 471 static const VMStateDescription vmstate_pmsav7_rnr = {
 472     .name = "cpu/pmsav7-rnr",
 473     .version_id = 1,
 474     .minimum_version_id = 1,
 475     .needed = pmsav7_rnr_needed,
 476     .fields = (VMStateField[]) {
 477         VMSTATE_UINT32(env.pmsav7.rnr[M_REG_NS], ARMCPU),
 478         VMSTATE_END_OF_LIST()
 479     }
 480 };
 481
 482 static bool pmsav8_needed(void *opaque)
 483 {
 484     ARMCPU *cpu = opaque;
 485     CPUARMState *env = &cpu->env;
 486
 487     return arm_feature(env, ARM_FEATURE_PMSA) &&
 488         arm_feature(env, ARM_FEATURE_V8);
 489 }
 490
 491 static bool pmsav8r_needed(void *opaque)
 492 {
 493     ARMCPU *cpu = opaque;
 494     CPUARMState *env = &cpu->env;
 495
 496     return arm_feature(env, ARM_FEATURE_PMSA) &&
 497         arm_feature(env, ARM_FEATURE_V8) &&
 498         !arm_feature(env, ARM_FEATURE_M);
 499 }
 500
 501 static const VMStateDescription vmstate_pmsav8r = {
 502     .name = "cpu/pmsav8/pmsav8r",
 503     .version_id = 1,
 504     .minimum_version_id = 1,
 505     .needed = pmsav8r_needed,
 506     .fields = (VMStateField[]) {
 507         VMSTATE_VARRAY_UINT32(env.pmsav8.hprbar, ARMCPU,
 508                         pmsav8r_hdregion, 0, vmstate_info_uint32, uint32_t),
 509         VMSTATE_VARRAY_UINT32(env.pmsav8.hprlar, ARMCPU,
 510                         pmsav8r_hdregion, 0, vmstate_info_uint32, uint32_t),
 511         VMSTATE_END_OF_LIST()
 512     },
 513 };
 514
 515 static const VMStateDescription vmstate_pmsav8 = {
 516     .name = "cpu/pmsav8",
 517     .version_id = 1,
 518     .minimum_version_id = 1,
 519     .needed = pmsav8_needed,
 520     .fields = (VMStateField[]) {
 521         VMSTATE_VARRAY_UINT32(env.pmsav8.rbar[M_REG_NS], ARMCPU, pmsav7_dregion,
 522                               0, vmstate_info_uint32, uint32_t),
 523         VMSTATE_VARRAY_UINT32(env.pmsav8.rlar[M_REG_NS], ARMCPU, pmsav7_dregion,
 524                               0, vmstate_info_uint32, uint32_t),
 525         VMSTATE_UINT32(env.pmsav8.mair0[M_REG_NS], ARMCPU),
 526         VMSTATE_UINT32(env.pmsav8.mair1[M_REG_NS], ARMCPU),
 527         VMSTATE_END_OF_LIST()
 528     },
 529     .subsections = (const VMStateDescription * []) {
 530         &vmstate_pmsav8r,
 531         NULL
 532     }
 533 };
 534
 535 static bool s_rnr_vmstate_validate(void *opaque, int version_id)
 536 {
 537     ARMCPU *cpu = opaque;
 538
 539     return cpu->env.pmsav7.rnr[M_REG_S] < cpu->pmsav7_dregion;
 540 }
 541
 542 static bool sau_rnr_vmstate_validate(void *opaque, int version_id)
 543 {
 544     ARMCPU *cpu = opaque;
 545
 546     return cpu->env.sau.rnr < cpu->sau_sregion;
 547 }
 548
 549 static bool m_security_needed(void *opaque)
 550 {
 551     ARMCPU *cpu = opaque;
 552     CPUARMState *env = &cpu->env;
 553
 554     return arm_feature(env, ARM_FEATURE_M_SECURITY);
 555 }
 556
 557 static const VMStateDescription vmstate_m_security = {
 558     .name = "cpu/m-security",
 559     .version_id = 1,
 560     .minimum_version_id = 1,
 561     .needed = m_security_needed,
 562     .fields = (VMStateField[]) {
 563         VMSTATE_UINT32(env.v7m.secure, ARMCPU),
 564         VMSTATE_UINT32(env.v7m.other_ss_msp, ARMCPU),
 565         VMSTATE_UINT32(env.v7m.other_ss_psp, ARMCPU),
 566         VMSTATE_UINT32(env.v7m.basepri[M_REG_S], ARMCPU),
 567         VMSTATE_UINT32(env.v7m.primask[M_REG_S], ARMCPU),
 568         VMSTATE_UINT32(env.v7m.faultmask[M_REG_S], ARMCPU),
 569         VMSTATE_UINT32(env.v7m.control[M_REG_S], ARMCPU),
 570         VMSTATE_UINT32(env.v7m.vecbase[M_REG_S], ARMCPU),
 571         VMSTATE_UINT32(env.pmsav8.mair0[M_REG_S], ARMCPU),
 572         VMSTATE_UINT32(env.pmsav8.mair1[M_REG_S], ARMCPU),
 573         VMSTATE_VARRAY_UINT32(env.pmsav8.rbar[M_REG_S], ARMCPU, pmsav7_dregion,
 574                               0, vmstate_info_uint32, uint32_t),
 575         VMSTATE_VARRAY_UINT32(env.pmsav8.rlar[M_REG_S], ARMCPU, pmsav7_dregion,
 576                               0, vmstate_info_uint32, uint32_t),
 577         VMSTATE_UINT32(env.pmsav7.rnr[M_REG_S], ARMCPU),
 578         VMSTATE_VALIDATE("secure MPU_RNR is valid", s_rnr_vmstate_validate),
 579         VMSTATE_UINT32(env.v7m.mpu_ctrl[M_REG_S], ARMCPU),
 580         VMSTATE_UINT32(env.v7m.ccr[M_REG_S], ARMCPU),
 581         VMSTATE_UINT32(env.v7m.mmfar[M_REG_S], ARMCPU),
 582         VMSTATE_UINT32(env.v7m.cfsr[M_REG_S], ARMCPU),
 583         VMSTATE_UINT32(env.v7m.sfsr, ARMCPU),
 584         VMSTATE_UINT32(env.v7m.sfar, ARMCPU),
 585         VMSTATE_VARRAY_UINT32(env.sau.rbar, ARMCPU, sau_sregion, 0,
 586                               vmstate_info_uint32, uint32_t),
 587         VMSTATE_VARRAY_UINT32(env.sau.rlar, ARMCPU, sau_sregion, 0,
 588                               vmstate_info_uint32, uint32_t),
 589         VMSTATE_UINT32(env.sau.rnr, ARMCPU),
 590         VMSTATE_VALIDATE("SAU_RNR is valid", sau_rnr_vmstate_validate),
 591         VMSTATE_UINT32(env.sau.ctrl, ARMCPU),
 592         VMSTATE_UINT32(env.v7m.scr[M_REG_S], ARMCPU),
 593         /* AIRCR is not secure-only, but our implementation is R/O if the
 594          * security extension is unimplemented, so we migrate it here.
 595          */
 596         VMSTATE_UINT32(env.v7m.aircr, ARMCPU),
 597         VMSTATE_END_OF_LIST()
 598     }
 599 };
 600
 601 static int get_cpsr(QEMUFile *f, void *opaque, size_t size,
 602                     const VMStateField *field)
 603 {
 604     ARMCPU *cpu = opaque;
 605     CPUARMState *env = &cpu->env;
 606     uint32_t val = qemu_get_be32(f);
 607
 608     if (arm_feature(env, ARM_FEATURE_M)) {
 609         if (val & XPSR_EXCP) {
 610             /* This is a CPSR format value from an older QEMU. (We can tell
 611              * because values transferred in XPSR format always have zero
 612              * for the EXCP field, and CPSR format will always have bit 4
 613              * set in CPSR_M.) Rearrange it into XPSR format. The significant
 614              * differences are that the T bit is not in the same place, the
 615              * primask/faultmask info may be in the CPSR I and F bits, and
 616              * we do not want the mode bits.
 617              * We know that this cleanup happened before v8M, so there
 618              * is no complication with banked primask/faultmask.
 619              */
 620             uint32_t newval = val;
 621
 622             assert(!arm_feature(env, ARM_FEATURE_M_SECURITY));
 623
 624             newval &= (CPSR_NZCV | CPSR_Q | CPSR_IT | CPSR_GE);
 625             if (val & CPSR_T) {
 626                 newval |= XPSR_T;
 627             }
 628             /* If the I or F bits are set then this is a migration from
 629              * an old QEMU which still stored the M profile FAULTMASK
 630              * and PRIMASK in env->daif. For a new QEMU, the data is
 631              * transferred using the vmstate_m_faultmask_primask subsection.
 632              */
 633             if (val & CPSR_F) {
 634                 env->v7m.faultmask[M_REG_NS] = 1;
 635             }
 636             if (val & CPSR_I) {
 637                 env->v7m.primask[M_REG_NS] = 1;
 638             }
 639             val = newval;
 640         }
 641         /* Ignore the low bits, they are handled by vmstate_m. */
 642         xpsr_write(env, val, ~XPSR_EXCP);
 643         return 0;
 644     }
 645
 646     env->aarch64 = ((val & PSTATE_nRW) == 0);
 647
 648     if (is_a64(env)) {
 649         pstate_write(env, val);
 650         return 0;
 651     }
 652
 653     cpsr_write(env, val, 0xffffffff, CPSRWriteRaw);
 654     return 0;
 655 }
 656
 657 static int put_cpsr(QEMUFile *f, void *opaque, size_t size,
 658                     const VMStateField *field, JSONWriter *vmdesc)
 659 {
 660     ARMCPU *cpu = opaque;
 661     CPUARMState *env = &cpu->env;
 662     uint32_t val;
 663
 664     if (arm_feature(env, ARM_FEATURE_M)) {
 665         /* The low 9 bits are v7m.exception, which is handled by vmstate_m. */
 666         val = xpsr_read(env) & ~XPSR_EXCP;
 667     } else if (is_a64(env)) {
 668         val = pstate_read(env);
 669     } else {
 670         val = cpsr_read(env);
 671     }
 672
 673     qemu_put_be32(f, val);
 674     return 0;
 675 }
 676
 677 static const VMStateInfo vmstate_cpsr = {
 678     .name = "cpsr",
 679     .get = get_cpsr,
 680     .put = put_cpsr,
 681 };
 682
 683 static int get_power(QEMUFile *f, void *opaque, size_t size,
 684                     const VMStateField *field)
 685 {
 686     ARMCPU *cpu = opaque;
 687     bool powered_off = qemu_get_byte(f);
 688     cpu->power_state = powered_off ? PSCI_OFF : PSCI_ON;
 689     return 0;
 690 }
 691
 692 static int put_power(QEMUFile *f, void *opaque, size_t size,
 693                     const VMStateField *field, JSONWriter *vmdesc)
 694 {
 695     ARMCPU *cpu = opaque;
 696
 697     /* Migration should never happen while we transition power states */
 698
 699     if (cpu->power_state == PSCI_ON ||
 700         cpu->power_state == PSCI_OFF) {
 701         bool powered_off = (cpu->power_state == PSCI_OFF) ? true : false;
 702         qemu_put_byte(f, powered_off);
 703         return 0;
 704     } else {
 705         return 1;
 706     }
 707 }
 708
 709 static const VMStateInfo vmstate_powered_off = {
 710     .name = "powered_off",
 711     .get = get_power,
 712     .put = put_power,
 713 };
 714
 715 static int cpu_pre_save(void *opaque)
 716 {
 717     ARMCPU *cpu = opaque;
 718
 719     if (!kvm_enabled()) {
 720         pmu_op_start(&cpu->env);
 721     }
 722
 723     if (kvm_enabled()) {
 724         if (!write_kvmstate_to_list(cpu)) {
 725             /* This should never fail */
 726             g_assert_not_reached();
 727         }
 728
 729         /*
 730          * kvm_arm_cpu_pre_save() must be called after
 731          * write_kvmstate_to_list()
 732          */
 733         kvm_arm_cpu_pre_save(cpu);
 734     } else {
 735         if (!write_cpustate_to_list(cpu, false)) {
 736             /* This should never fail. */
 737             g_assert_not_reached();
 738         }
 739     }
 740
 741     cpu->cpreg_vmstate_array_len = cpu->cpreg_array_len;
 742     memcpy(cpu->cpreg_vmstate_indexes, cpu->cpreg_indexes,
 743            cpu->cpreg_array_len * sizeof(uint64_t));
 744     memcpy(cpu->cpreg_vmstate_values, cpu->cpreg_values,
 745            cpu->cpreg_array_len * sizeof(uint64_t));
 746
 747     return 0;
 748 }
 749
 750 static int cpu_post_save(void *opaque)
 751 {
 752     ARMCPU *cpu = opaque;
 753
 754     if (!kvm_enabled()) {
 755         pmu_op_finish(&cpu->env);
 756     }
 757
 758     return 0;
 759 }
 760
 761 static int cpu_pre_load(void *opaque)
 762 {
 763     ARMCPU *cpu = opaque;
 764     CPUARMState *env = &cpu->env;
 765
 766     /*
 767      * Pre-initialize irq_line_state to a value that's never valid as
 768      * real data, so cpu_post_load() can tell whether we've seen the
 769      * irq-line-state subsection in the incoming migration state.
 770      */
 771     env->irq_line_state = UINT32_MAX;
 772
 773     if (!kvm_enabled()) {
 774         pmu_op_start(&cpu->env);
 775     }
 776
 777     return 0;
 778 }
 779
 780 static int cpu_post_load(void *opaque, int version_id)
 781 {
 782     ARMCPU *cpu = opaque;
 783     CPUARMState *env = &cpu->env;
 784     int i, v;
 785
 786     /*
 787      * Handle migration compatibility from old QEMU which didn't
 788      * send the irq-line-state subsection. A QEMU without it did not
 789      * implement the HCR_EL2.{VI,VF} bits as generating interrupts,
 790      * so for TCG the line state matches the bits set in cs->interrupt_request.
 791      * For KVM the line state is not stored in cs->interrupt_request
 792      * and so this will leave irq_line_state as 0, but this is OK because
 793      * we only need to care about it for TCG.
 794      */
 795     if (env->irq_line_state == UINT32_MAX) {
 796         CPUState *cs = CPU(cpu);
 797
 798         env->irq_line_state = cs->interrupt_request &
 799             (CPU_INTERRUPT_HARD | CPU_INTERRUPT_FIQ |
 800              CPU_INTERRUPT_VIRQ | CPU_INTERRUPT_VFIQ);
 801     }
 802
 803     /* Update the values list from the incoming migration data.
 804      * Anything in the incoming data which we don't know about is
 805      * a migration failure; anything we know about but the incoming
 806      * data doesn't specify retains its current (reset) value.
 807      * The indexes list remains untouched -- we only inspect the
 808      * incoming migration index list so we can match the values array
 809      * entries with the right slots in our own values array.
 810      */
 811
 812     for (i = 0, v = 0; i < cpu->cpreg_array_len
 813              && v < cpu->cpreg_vmstate_array_len; i++) {
 814         if (cpu->cpreg_vmstate_indexes[v] > cpu->cpreg_indexes[i]) {
 815             /* register in our list but not incoming : skip it */
 816             continue;
 817         }
 818         if (cpu->cpreg_vmstate_indexes[v] < cpu->cpreg_indexes[i]) {
 819             /* register in their list but not ours: fail migration */
 820             return -1;
 821         }
 822         /* matching register, copy the value over */
 823         cpu->cpreg_values[i] = cpu->cpreg_vmstate_values[v];
 824         v++;
 825     }
 826
 827     if (kvm_enabled()) {
 828         if (!write_list_to_kvmstate(cpu, KVM_PUT_FULL_STATE)) {
 829             return -1;
 830         }
 831         /* Note that it's OK for the TCG side not to know about
 832          * every register in the list; KVM is authoritative if
 833          * we're using it.
 834          */
 835         write_list_to_cpustate(cpu);
 836         kvm_arm_cpu_post_load(cpu);
 837     } else {
 838         if (!write_list_to_cpustate(cpu)) {
 839             return -1;
 840         }
 841     }
 842
 843     /*
 844      * Misaligned thumb pc is architecturally impossible. Fail the
 845      * incoming migration. For TCG it would trigger the assert in
 846      * thumb_tr_translate_insn().
 847      */
 848     if (!is_a64(env) && env->thumb && (env->regs[15] & 1)) {
 849         return -1;
 850     }
 851
 852     if (tcg_enabled()) {
 853         hw_breakpoint_update_all(cpu);
 854         hw_watchpoint_update_all(cpu);
 855     }
 856
 857     /*
 858      * TCG gen_update_fp_context() relies on the invariant that
 859      * FPDSCR.LTPSIZE is constant 4 for M-profile with the LOB extension;
 860      * forbid bogus incoming data with some other value.
 861      */
 862     if (arm_feature(env, ARM_FEATURE_M) && cpu_isar_feature(aa32_lob, cpu)) {
 863         if (extract32(env->v7m.fpdscr[M_REG_NS],
 864                       FPCR_LTPSIZE_SHIFT, FPCR_LTPSIZE_LENGTH) != 4 ||
 865             extract32(env->v7m.fpdscr[M_REG_S],
 866                       FPCR_LTPSIZE_SHIFT, FPCR_LTPSIZE_LENGTH) != 4) {
 867             return -1;
 868         }
 869     }
 870
 871     if (!kvm_enabled()) {
 872         pmu_op_finish(&cpu->env);
 873     }
 874
 875     if (tcg_enabled()) {
 876         arm_rebuild_hflags(&cpu->env);
 877     }
 878
 879     return 0;
 880 }
 881
 882 const VMStateDescription vmstate_arm_cpu = {
 883     .name = "cpu",
 884     .version_id = 22,
 885     .minimum_version_id = 22,
 886     .pre_save = cpu_pre_save,
 887     .post_save = cpu_post_save,
 888     .pre_load = cpu_pre_load,
 889     .post_load = cpu_post_load,
 890     .fields = (VMStateField[]) {
 891         VMSTATE_UINT32_ARRAY(env.regs, ARMCPU, 16),
 892         VMSTATE_UINT64_ARRAY(env.xregs, ARMCPU, 32),
 893         VMSTATE_UINT64(env.pc, ARMCPU),
 894         {
 895             .name = "cpsr",
 896             .version_id = 0,
 897             .size = sizeof(uint32_t),
 898             .info = &vmstate_cpsr,
 899             .flags = VMS_SINGLE,
 900             .offset = 0,
 901         },
 902         VMSTATE_UINT32(env.spsr, ARMCPU),
 903         VMSTATE_UINT64_ARRAY(env.banked_spsr, ARMCPU, 8),
 904         VMSTATE_UINT32_ARRAY(env.banked_r13, ARMCPU, 8),
 905         VMSTATE_UINT32_ARRAY(env.banked_r14, ARMCPU, 8),
 906         VMSTATE_UINT32_ARRAY(env.usr_regs, ARMCPU, 5),
 907         VMSTATE_UINT32_ARRAY(env.fiq_regs, ARMCPU, 5),
 908         VMSTATE_UINT64_ARRAY(env.elr_el, ARMCPU, 4),
 909         VMSTATE_UINT64_ARRAY(env.sp_el, ARMCPU, 4),
 910         /* The length-check must come before the arrays to avoid
 911          * incoming data possibly overflowing the array.
 912          */
 913         VMSTATE_INT32_POSITIVE_LE(cpreg_vmstate_array_len, ARMCPU),
 914         VMSTATE_VARRAY_INT32(cpreg_vmstate_indexes, ARMCPU,
 915                              cpreg_vmstate_array_len,
 916                              0, vmstate_info_uint64, uint64_t),
 917         VMSTATE_VARRAY_INT32(cpreg_vmstate_values, ARMCPU,
 918                              cpreg_vmstate_array_len,
 919                              0, vmstate_info_uint64, uint64_t),
 920         VMSTATE_UINT64(env.exclusive_addr, ARMCPU),
 921         VMSTATE_UINT64(env.exclusive_val, ARMCPU),
 922         VMSTATE_UINT64(env.exclusive_high, ARMCPU),
 923         VMSTATE_UNUSED(sizeof(uint64_t)),
 924         VMSTATE_UINT32(env.exception.syndrome, ARMCPU),
 925         VMSTATE_UINT32(env.exception.fsr, ARMCPU),
 926         VMSTATE_UINT64(env.exception.vaddress, ARMCPU),
 927         VMSTATE_TIMER_PTR(gt_timer[GTIMER_PHYS], ARMCPU),
 928         VMSTATE_TIMER_PTR(gt_timer[GTIMER_VIRT], ARMCPU),
 929         {
 930             .name = "power_state",
 931             .version_id = 0,
 932             .size = sizeof(bool),
 933             .info = &vmstate_powered_off,
 934             .flags = VMS_SINGLE,
 935             .offset = 0,
 936         },
 937         VMSTATE_END_OF_LIST()
 938     },
 939     .subsections = (const VMStateDescription*[]) {
 940         &vmstate_vfp,
 941         &vmstate_iwmmxt,
 942         &vmstate_m,
 943         &vmstate_thumb2ee,
 944         /* pmsav7_rnr must come before pmsav7 so that we have the
 945          * region number before we test it in the VMSTATE_VALIDATE
 946          * in vmstate_pmsav7.
 947          */
 948         &vmstate_pmsav7_rnr,
 949         &vmstate_pmsav7,
 950         &vmstate_pmsav8,
 951         &vmstate_m_security,
 952 #ifdef TARGET_AARCH64
 953         &vmstate_sve,
 954         &vmstate_za,
 955 #endif
 956         &vmstate_serror,
 957         &vmstate_irq_line_state,
 958         NULL
 959     }
 960 };