hw/ppc/spapr_hcall.c

   1 #include "qemu/osdep.h"
   2 #include "qapi/error.h"
   3 #include "sysemu/hw_accel.h"
   4 #include "sysemu/sysemu.h"
   5 #include "qemu/log.h"
   6 #include "qemu/error-report.h"
   7 #include "cpu.h"
   8 #include "exec/exec-all.h"
   9 #include "helper_regs.h"
  10 #include "hw/ppc/spapr.h"
  11 #include "hw/ppc/spapr_cpu_core.h"
  12 #include "mmu-hash64.h"
  13 #include "cpu-models.h"
  14 #include "trace.h"
  15 #include "kvm_ppc.h"
  16 #include "hw/ppc/spapr_ovec.h"
  17 #include "mmu-book3s-v3.h"
  18 #include "hw/mem/memory-device.h"
  19
  20 struct LPCRSyncState {
  21     target_ulong value;
  22     target_ulong mask;
  23 };
  24
  25 static void do_lpcr_sync(CPUState *cs, run_on_cpu_data arg)
  26 {
  27     struct LPCRSyncState *s = arg.host_ptr;
  28     PowerPCCPU *cpu = POWERPC_CPU(cs);
  29     CPUPPCState *env = &cpu->env;
  30     target_ulong lpcr;
  31
  32     cpu_synchronize_state(cs);
  33     lpcr = env->spr[SPR_LPCR];
  34     lpcr &= ~s->mask;
  35     lpcr |= s->value;
  36     ppc_store_lpcr(cpu, lpcr);
  37 }
  38
  39 static void set_all_lpcrs(target_ulong value, target_ulong mask)
  40 {
  41     CPUState *cs;
  42     struct LPCRSyncState s = {
  43         .value = value,
  44         .mask = mask
  45     };
  46     CPU_FOREACH(cs) {
  47         run_on_cpu(cs, do_lpcr_sync, RUN_ON_CPU_HOST_PTR(&s));
  48     }
  49 }
  50
  51 static bool has_spr(PowerPCCPU *cpu, int spr)
  52 {
  53     /* We can test whether the SPR is defined by checking for a valid name */
  54     return cpu->env.spr_cb[spr].name != NULL;
  55 }
  56
  57 static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
  58 {
  59     /*
  60      * hash value/pteg group index is normalized by HPT mask
  61      */
  62     if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
  63         return false;
  64     }
  65     return true;
  66 }
  67
  68 static bool is_ram_address(sPAPRMachineState *spapr, hwaddr addr)
  69 {
  70     MachineState *machine = MACHINE(spapr);
  71     DeviceMemoryState *dms = machine->device_memory;
  72
  73     if (addr < machine->ram_size) {
  74         return true;
  75     }
  76     if ((addr >= dms->base)
  77         && ((addr - dms->base) < memory_region_size(&dms->mr))) {
  78         return true;
  79     }
  80
  81     return false;
  82 }
  83
  84 static target_ulong h_enter(PowerPCCPU *cpu, sPAPRMachineState *spapr,
  85                             target_ulong opcode, target_ulong *args)
  86 {
  87     target_ulong flags = args[0];
  88     target_ulong ptex = args[1];
  89     target_ulong pteh = args[2];
  90     target_ulong ptel = args[3];
  91     unsigned apshift;
  92     target_ulong raddr;
  93     target_ulong slot;
  94     const ppc_hash_pte64_t *hptes;
  95
  96     apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
  97     if (!apshift) {
  98         /* Bad page size encoding */
  99         return H_PARAMETER;
 100     }
 101
 102     raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
 103
 104     if (is_ram_address(spapr, raddr)) {
 105         /* Regular RAM - should have WIMG=0010 */
 106         if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
 107             return H_PARAMETER;
 108         }
 109     } else {
 110         target_ulong wimg_flags;
 111         /* Looks like an IO address */
 112         /* FIXME: What WIMG combinations could be sensible for IO?
 113          * For now we allow WIMG=010x, but are there others? */
 114         /* FIXME: Should we check against registered IO addresses? */
 115         wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
 116
 117         if (wimg_flags != HPTE64_R_I &&
 118             wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
 119             return H_PARAMETER;
 120         }
 121     }
 122
 123     pteh &= ~0x60ULL;
 124
 125     if (!valid_ptex(cpu, ptex)) {
 126         return H_PARAMETER;
 127     }
 128
 129     slot = ptex & 7ULL;
 130     ptex = ptex & ~7ULL;
 131
 132     if (likely((flags & H_EXACT) == 0)) {
 133         hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
 134         for (slot = 0; slot < 8; slot++) {
 135             if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
 136                 break;
 137             }
 138         }
 139         ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
 140         if (slot == 8) {
 141             return H_PTEG_FULL;
 142         }
 143     } else {
 144         hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
 145         if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
 146             ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
 147             return H_PTEG_FULL;
 148         }
 149         ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
 150     }
 151
 152     ppc_hash64_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
 153
 154     args[0] = ptex + slot;
 155     return H_SUCCESS;
 156 }
 157
 158 typedef enum {
 159     REMOVE_SUCCESS = 0,
 160     REMOVE_NOT_FOUND = 1,
 161     REMOVE_PARM = 2,
 162     REMOVE_HW = 3,
 163 } RemoveResult;
 164
 165 static RemoveResult remove_hpte(PowerPCCPU *cpu, target_ulong ptex,
 166                                 target_ulong avpn,
 167                                 target_ulong flags,
 168                                 target_ulong *vp, target_ulong *rp)
 169 {
 170     const ppc_hash_pte64_t *hptes;
 171     target_ulong v, r;
 172
 173     if (!valid_ptex(cpu, ptex)) {
 174         return REMOVE_PARM;
 175     }
 176
 177     hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
 178     v = ppc_hash64_hpte0(cpu, hptes, 0);
 179     r = ppc_hash64_hpte1(cpu, hptes, 0);
 180     ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
 181
 182     if ((v & HPTE64_V_VALID) == 0 ||
 183         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
 184         ((flags & H_ANDCOND) && (v & avpn) != 0)) {
 185         return REMOVE_NOT_FOUND;
 186     }
 187     *vp = v;
 188     *rp = r;
 189     ppc_hash64_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
 190     ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
 191     return REMOVE_SUCCESS;
 192 }
 193
 194 static target_ulong h_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 195                              target_ulong opcode, target_ulong *args)
 196 {
 197     CPUPPCState *env = &cpu->env;
 198     target_ulong flags = args[0];
 199     target_ulong ptex = args[1];
 200     target_ulong avpn = args[2];
 201     RemoveResult ret;
 202
 203     ret = remove_hpte(cpu, ptex, avpn, flags,
 204                       &args[0], &args[1]);
 205
 206     switch (ret) {
 207     case REMOVE_SUCCESS:
 208         check_tlb_flush(env, true);
 209         return H_SUCCESS;
 210
 211     case REMOVE_NOT_FOUND:
 212         return H_NOT_FOUND;
 213
 214     case REMOVE_PARM:
 215         return H_PARAMETER;
 216
 217     case REMOVE_HW:
 218         return H_HARDWARE;
 219     }
 220
 221     g_assert_not_reached();
 222 }
 223
 224 #define H_BULK_REMOVE_TYPE             0xc000000000000000ULL
 225 #define   H_BULK_REMOVE_REQUEST        0x4000000000000000ULL
 226 #define   H_BULK_REMOVE_RESPONSE       0x8000000000000000ULL
 227 #define   H_BULK_REMOVE_END            0xc000000000000000ULL
 228 #define H_BULK_REMOVE_CODE             0x3000000000000000ULL
 229 #define   H_BULK_REMOVE_SUCCESS        0x0000000000000000ULL
 230 #define   H_BULK_REMOVE_NOT_FOUND      0x1000000000000000ULL
 231 #define   H_BULK_REMOVE_PARM           0x2000000000000000ULL
 232 #define   H_BULK_REMOVE_HW             0x3000000000000000ULL
 233 #define H_BULK_REMOVE_RC               0x0c00000000000000ULL
 234 #define H_BULK_REMOVE_FLAGS            0x0300000000000000ULL
 235 #define   H_BULK_REMOVE_ABSOLUTE       0x0000000000000000ULL
 236 #define   H_BULK_REMOVE_ANDCOND        0x0100000000000000ULL
 237 #define   H_BULK_REMOVE_AVPN           0x0200000000000000ULL
 238 #define H_BULK_REMOVE_PTEX             0x00ffffffffffffffULL
 239
 240 #define H_BULK_REMOVE_MAX_BATCH        4
 241
 242 static target_ulong h_bulk_remove(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 243                                   target_ulong opcode, target_ulong *args)
 244 {
 245     CPUPPCState *env = &cpu->env;
 246     int i;
 247     target_ulong rc = H_SUCCESS;
 248
 249     for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
 250         target_ulong *tsh = &args[i*2];
 251         target_ulong tsl = args[i*2 + 1];
 252         target_ulong v, r, ret;
 253
 254         if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
 255             break;
 256         } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
 257             return H_PARAMETER;
 258         }
 259
 260         *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
 261         *tsh |= H_BULK_REMOVE_RESPONSE;
 262
 263         if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
 264             *tsh |= H_BULK_REMOVE_PARM;
 265             return H_PARAMETER;
 266         }
 267
 268         ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
 269                           (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
 270                           &v, &r);
 271
 272         *tsh |= ret << 60;
 273
 274         switch (ret) {
 275         case REMOVE_SUCCESS:
 276             *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
 277             break;
 278
 279         case REMOVE_PARM:
 280             rc = H_PARAMETER;
 281             goto exit;
 282
 283         case REMOVE_HW:
 284             rc = H_HARDWARE;
 285             goto exit;
 286         }
 287     }
 288  exit:
 289     check_tlb_flush(env, true);
 290
 291     return rc;
 292 }
 293
 294 static target_ulong h_protect(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 295                               target_ulong opcode, target_ulong *args)
 296 {
 297     CPUPPCState *env = &cpu->env;
 298     target_ulong flags = args[0];
 299     target_ulong ptex = args[1];
 300     target_ulong avpn = args[2];
 301     const ppc_hash_pte64_t *hptes;
 302     target_ulong v, r;
 303
 304     if (!valid_ptex(cpu, ptex)) {
 305         return H_PARAMETER;
 306     }
 307
 308     hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
 309     v = ppc_hash64_hpte0(cpu, hptes, 0);
 310     r = ppc_hash64_hpte1(cpu, hptes, 0);
 311     ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
 312
 313     if ((v & HPTE64_V_VALID) == 0 ||
 314         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
 315         return H_NOT_FOUND;
 316     }
 317
 318     r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
 319            HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
 320     r |= (flags << 55) & HPTE64_R_PP0;
 321     r |= (flags << 48) & HPTE64_R_KEY_HI;
 322     r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
 323     ppc_hash64_store_hpte(cpu, ptex,
 324                           (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
 325     ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
 326     /* Flush the tlb */
 327     check_tlb_flush(env, true);
 328     /* Don't need a memory barrier, due to qemu's global lock */
 329     ppc_hash64_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
 330     return H_SUCCESS;
 331 }
 332
 333 static target_ulong h_read(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 334                            target_ulong opcode, target_ulong *args)
 335 {
 336     target_ulong flags = args[0];
 337     target_ulong ptex = args[1];
 338     uint8_t *hpte;
 339     int i, ridx, n_entries = 1;
 340
 341     if (!valid_ptex(cpu, ptex)) {
 342         return H_PARAMETER;
 343     }
 344
 345     if (flags & H_READ_4) {
 346         /* Clear the two low order bits */
 347         ptex &= ~(3ULL);
 348         n_entries = 4;
 349     }
 350
 351     hpte = spapr->htab + (ptex * HASH_PTE_SIZE_64);
 352
 353     for (i = 0, ridx = 0; i < n_entries; i++) {
 354         args[ridx++] = ldq_p(hpte);
 355         args[ridx++] = ldq_p(hpte + (HASH_PTE_SIZE_64/2));
 356         hpte += HASH_PTE_SIZE_64;
 357     }
 358
 359     return H_SUCCESS;
 360 }
 361
 362 struct sPAPRPendingHPT {
 363     /* These fields are read-only after initialization */
 364     int shift;
 365     QemuThread thread;
 366
 367     /* These fields are protected by the BQL */
 368     bool complete;
 369
 370     /* These fields are private to the preparation thread if
 371      * !complete, otherwise protected by the BQL */
 372     int ret;
 373     void *hpt;
 374 };
 375
 376 static void free_pending_hpt(sPAPRPendingHPT *pending)
 377 {
 378     if (pending->hpt) {
 379         qemu_vfree(pending->hpt);
 380     }
 381
 382     g_free(pending);
 383 }
 384
 385 static void *hpt_prepare_thread(void *opaque)
 386 {
 387     sPAPRPendingHPT *pending = opaque;
 388     size_t size = 1ULL << pending->shift;
 389
 390     pending->hpt = qemu_memalign(size, size);
 391     if (pending->hpt) {
 392         memset(pending->hpt, 0, size);
 393         pending->ret = H_SUCCESS;
 394     } else {
 395         pending->ret = H_NO_MEM;
 396     }
 397
 398     qemu_mutex_lock_iothread();
 399
 400     if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
 401         /* Ready to go */
 402         pending->complete = true;
 403     } else {
 404         /* We've been cancelled, clean ourselves up */
 405         free_pending_hpt(pending);
 406     }
 407
 408     qemu_mutex_unlock_iothread();
 409     return NULL;
 410 }
 411
 412 /* Must be called with BQL held */
 413 static void cancel_hpt_prepare(sPAPRMachineState *spapr)
 414 {
 415     sPAPRPendingHPT *pending = spapr->pending_hpt;
 416
 417     /* Let the thread know it's cancelled */
 418     spapr->pending_hpt = NULL;
 419
 420     if (!pending) {
 421         /* Nothing to do */
 422         return;
 423     }
 424
 425     if (!pending->complete) {
 426         /* thread will clean itself up */
 427         return;
 428     }
 429
 430     free_pending_hpt(pending);
 431 }
 432
 433 /* Convert a return code from the KVM ioctl()s implementing resize HPT
 434  * into a PAPR hypercall return code */
 435 static target_ulong resize_hpt_convert_rc(int ret)
 436 {
 437     if (ret >= 100000) {
 438         return H_LONG_BUSY_ORDER_100_SEC;
 439     } else if (ret >= 10000) {
 440         return H_LONG_BUSY_ORDER_10_SEC;
 441     } else if (ret >= 1000) {
 442         return H_LONG_BUSY_ORDER_1_SEC;
 443     } else if (ret >= 100) {
 444         return H_LONG_BUSY_ORDER_100_MSEC;
 445     } else if (ret >= 10) {
 446         return H_LONG_BUSY_ORDER_10_MSEC;
 447     } else if (ret > 0) {
 448         return H_LONG_BUSY_ORDER_1_MSEC;
 449     }
 450
 451     switch (ret) {
 452     case 0:
 453         return H_SUCCESS;
 454     case -EPERM:
 455         return H_AUTHORITY;
 456     case -EINVAL:
 457         return H_PARAMETER;
 458     case -ENXIO:
 459         return H_CLOSED;
 460     case -ENOSPC:
 461         return H_PTEG_FULL;
 462     case -EBUSY:
 463         return H_BUSY;
 464     case -ENOMEM:
 465         return H_NO_MEM;
 466     default:
 467         return H_HARDWARE;
 468     }
 469 }
 470
 471 static target_ulong h_resize_hpt_prepare(PowerPCCPU *cpu,
 472                                          sPAPRMachineState *spapr,
 473                                          target_ulong opcode,
 474                                          target_ulong *args)
 475 {
 476     target_ulong flags = args[0];
 477     int shift = args[1];
 478     sPAPRPendingHPT *pending = spapr->pending_hpt;
 479     uint64_t current_ram_size;
 480     int rc;
 481
 482     if (spapr->resize_hpt == SPAPR_RESIZE_HPT_DISABLED) {
 483         return H_AUTHORITY;
 484     }
 485
 486     if (!spapr->htab_shift) {
 487         /* Radix guest, no HPT */
 488         return H_NOT_AVAILABLE;
 489     }
 490
 491     trace_spapr_h_resize_hpt_prepare(flags, shift);
 492
 493     if (flags != 0) {
 494         return H_PARAMETER;
 495     }
 496
 497     if (shift && ((shift < 18) || (shift > 46))) {
 498         return H_PARAMETER;
 499     }
 500
 501     current_ram_size = MACHINE(spapr)->ram_size + get_plugged_memory_size();
 502
 503     /* We only allow the guest to allocate an HPT one order above what
 504      * we'd normally give them (to stop a small guest claiming a huge
 505      * chunk of resources in the HPT */
 506     if (shift > (spapr_hpt_shift_for_ramsize(current_ram_size) + 1)) {
 507         return H_RESOURCE;
 508     }
 509
 510     rc = kvmppc_resize_hpt_prepare(cpu, flags, shift);
 511     if (rc != -ENOSYS) {
 512         return resize_hpt_convert_rc(rc);
 513     }
 514
 515     if (pending) {
 516         /* something already in progress */
 517         if (pending->shift == shift) {
 518             /* and it's suitable */
 519             if (pending->complete) {
 520                 return pending->ret;
 521             } else {
 522                 return H_LONG_BUSY_ORDER_100_MSEC;
 523             }
 524         }
 525
 526         /* not suitable, cancel and replace */
 527         cancel_hpt_prepare(spapr);
 528     }
 529
 530     if (!shift) {
 531         /* nothing to do */
 532         return H_SUCCESS;
 533     }
 534
 535     /* start new prepare */
 536
 537     pending = g_new0(sPAPRPendingHPT, 1);
 538     pending->shift = shift;
 539     pending->ret = H_HARDWARE;
 540
 541     qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
 542                        hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
 543
 544     spapr->pending_hpt = pending;
 545
 546     /* In theory we could estimate the time more accurately based on
 547      * the new size, but there's not much point */
 548     return H_LONG_BUSY_ORDER_100_MSEC;
 549 }
 550
 551 static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
 552 {
 553     uint8_t *addr = htab;
 554
 555     addr += pteg * HASH_PTEG_SIZE_64;
 556     addr += slot * HASH_PTE_SIZE_64;
 557     return  ldq_p(addr);
 558 }
 559
 560 static void new_hpte_store(void *htab, uint64_t pteg, int slot,
 561                            uint64_t pte0, uint64_t pte1)
 562 {
 563     uint8_t *addr = htab;
 564
 565     addr += pteg * HASH_PTEG_SIZE_64;
 566     addr += slot * HASH_PTE_SIZE_64;
 567
 568     stq_p(addr, pte0);
 569     stq_p(addr + HASH_PTE_SIZE_64 / 2, pte1);
 570 }
 571
 572 static int rehash_hpte(PowerPCCPU *cpu,
 573                        const ppc_hash_pte64_t *hptes,
 574                        void *old_hpt, uint64_t oldsize,
 575                        void *new_hpt, uint64_t newsize,
 576                        uint64_t pteg, int slot)
 577 {
 578     uint64_t old_hash_mask = (oldsize >> 7) - 1;
 579     uint64_t new_hash_mask = (newsize >> 7) - 1;
 580     target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
 581     target_ulong pte1;
 582     uint64_t avpn;
 583     unsigned base_pg_shift;
 584     uint64_t hash, new_pteg, replace_pte0;
 585
 586     if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
 587         return H_SUCCESS;
 588     }
 589
 590     pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
 591
 592     base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
 593     assert(base_pg_shift); /* H_ENTER shouldn't allow a bad encoding */
 594     avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
 595
 596     if (pte0 & HPTE64_V_SECONDARY) {
 597         pteg = ~pteg;
 598     }
 599
 600     if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
 601         uint64_t offset, vsid;
 602
 603         /* We only have 28 - 23 bits of offset in avpn */
 604         offset = (avpn & 0x1f) << 23;
 605         vsid = avpn >> 5;
 606         /* We can find more bits from the pteg value */
 607         if (base_pg_shift < 23) {
 608             offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
 609         }
 610
 611         hash = vsid ^ (offset >> base_pg_shift);
 612     } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
 613         uint64_t offset, vsid;
 614
 615         /* We only have 40 - 23 bits of seg_off in avpn */
 616         offset = (avpn & 0x1ffff) << 23;
 617         vsid = avpn >> 17;
 618         if (base_pg_shift < 23) {
 619             offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
 620                 << base_pg_shift;
 621         }
 622
 623         hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
 624     } else {
 625         error_report("rehash_pte: Bad segment size in HPTE");
 626         return H_HARDWARE;
 627     }
 628
 629     new_pteg = hash & new_hash_mask;
 630     if (pte0 & HPTE64_V_SECONDARY) {
 631         assert(~pteg == (hash & old_hash_mask));
 632         new_pteg = ~new_pteg;
 633     } else {
 634         assert(pteg == (hash & old_hash_mask));
 635     }
 636     assert((oldsize != newsize) || (pteg == new_pteg));
 637     replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
 638     /*
 639      * Strictly speaking, we don't need all these tests, since we only
 640      * ever rehash bolted HPTEs.  We might in future handle non-bolted
 641      * HPTEs, though so make the logic correct for those cases as
 642      * well.
 643      */
 644     if (replace_pte0 & HPTE64_V_VALID) {
 645         assert(newsize < oldsize);
 646         if (replace_pte0 & HPTE64_V_BOLTED) {
 647             if (pte0 & HPTE64_V_BOLTED) {
 648                 /* Bolted collision, nothing we can do */
 649                 return H_PTEG_FULL;
 650             } else {
 651                 /* Discard this hpte */
 652                 return H_SUCCESS;
 653             }
 654         }
 655     }
 656
 657     new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
 658     return H_SUCCESS;
 659 }
 660
 661 static int rehash_hpt(PowerPCCPU *cpu,
 662                       void *old_hpt, uint64_t oldsize,
 663                       void *new_hpt, uint64_t newsize)
 664 {
 665     uint64_t n_ptegs = oldsize >> 7;
 666     uint64_t pteg;
 667     int slot;
 668     int rc;
 669
 670     for (pteg = 0; pteg < n_ptegs; pteg++) {
 671         hwaddr ptex = pteg * HPTES_PER_GROUP;
 672         const ppc_hash_pte64_t *hptes
 673             = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
 674
 675         if (!hptes) {
 676             return H_HARDWARE;
 677         }
 678
 679         for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
 680             rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
 681                              pteg, slot);
 682             if (rc != H_SUCCESS) {
 683                 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
 684                 return rc;
 685             }
 686         }
 687         ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
 688     }
 689
 690     return H_SUCCESS;
 691 }
 692
 693 static void do_push_sregs_to_kvm_pr(CPUState *cs, run_on_cpu_data data)
 694 {
 695     int ret;
 696
 697     cpu_synchronize_state(cs);
 698
 699     ret = kvmppc_put_books_sregs(POWERPC_CPU(cs));
 700     if (ret < 0) {
 701         error_report("failed to push sregs to KVM: %s", strerror(-ret));
 702         exit(1);
 703     }
 704 }
 705
 706 static void push_sregs_to_kvm_pr(sPAPRMachineState *spapr)
 707 {
 708     CPUState *cs;
 709
 710     /*
 711      * This is a hack for the benefit of KVM PR - it abuses the SDR1
 712      * slot in kvm_sregs to communicate the userspace address of the
 713      * HPT
 714      */
 715     if (!kvm_enabled() || !spapr->htab) {
 716         return;
 717     }
 718
 719     CPU_FOREACH(cs) {
 720         run_on_cpu(cs, do_push_sregs_to_kvm_pr, RUN_ON_CPU_NULL);
 721     }
 722 }
 723
 724 static target_ulong h_resize_hpt_commit(PowerPCCPU *cpu,
 725                                         sPAPRMachineState *spapr,
 726                                         target_ulong opcode,
 727                                         target_ulong *args)
 728 {
 729     target_ulong flags = args[0];
 730     target_ulong shift = args[1];
 731     sPAPRPendingHPT *pending = spapr->pending_hpt;
 732     int rc;
 733     size_t newsize;
 734
 735     if (spapr->resize_hpt == SPAPR_RESIZE_HPT_DISABLED) {
 736         return H_AUTHORITY;
 737     }
 738
 739     if (!spapr->htab_shift) {
 740         /* Radix guest, no HPT */
 741         return H_NOT_AVAILABLE;
 742     }
 743
 744     trace_spapr_h_resize_hpt_commit(flags, shift);
 745
 746     rc = kvmppc_resize_hpt_commit(cpu, flags, shift);
 747     if (rc != -ENOSYS) {
 748         rc = resize_hpt_convert_rc(rc);
 749         if (rc == H_SUCCESS) {
 750             /* Need to set the new htab_shift in the machine state */
 751             spapr->htab_shift = shift;
 752         }
 753         return rc;
 754     }
 755
 756     if (flags != 0) {
 757         return H_PARAMETER;
 758     }
 759
 760     if (!pending || (pending->shift != shift)) {
 761         /* no matching prepare */
 762         return H_CLOSED;
 763     }
 764
 765     if (!pending->complete) {
 766         /* prepare has not completed */
 767         return H_BUSY;
 768     }
 769
 770     /* Shouldn't have got past PREPARE without an HPT */
 771     g_assert(spapr->htab_shift);
 772
 773     newsize = 1ULL << pending->shift;
 774     rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
 775                     pending->hpt, newsize);
 776     if (rc == H_SUCCESS) {
 777         qemu_vfree(spapr->htab);
 778         spapr->htab = pending->hpt;
 779         spapr->htab_shift = pending->shift;
 780
 781         push_sregs_to_kvm_pr(spapr);
 782
 783         pending->hpt = NULL; /* so it's not free()d */
 784     }
 785
 786     /* Clean up */
 787     spapr->pending_hpt = NULL;
 788     free_pending_hpt(pending);
 789
 790     return rc;
 791 }
 792
 793 static target_ulong h_set_sprg0(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 794                                 target_ulong opcode, target_ulong *args)
 795 {
 796     cpu_synchronize_state(CPU(cpu));
 797     cpu->env.spr[SPR_SPRG0] = args[0];
 798
 799     return H_SUCCESS;
 800 }
 801
 802 static target_ulong h_set_dabr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 803                                target_ulong opcode, target_ulong *args)
 804 {
 805     if (!has_spr(cpu, SPR_DABR)) {
 806         return H_HARDWARE;              /* DABR register not available */
 807     }
 808     cpu_synchronize_state(CPU(cpu));
 809
 810     if (has_spr(cpu, SPR_DABRX)) {
 811         cpu->env.spr[SPR_DABRX] = 0x3;  /* Use Problem and Privileged state */
 812     } else if (!(args[0] & 0x4)) {      /* Breakpoint Translation set? */
 813         return H_RESERVED_DABR;
 814     }
 815
 816     cpu->env.spr[SPR_DABR] = args[0];
 817     return H_SUCCESS;
 818 }
 819
 820 static target_ulong h_set_xdabr(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 821                                 target_ulong opcode, target_ulong *args)
 822 {
 823     target_ulong dabrx = args[1];
 824
 825     if (!has_spr(cpu, SPR_DABR) || !has_spr(cpu, SPR_DABRX)) {
 826         return H_HARDWARE;
 827     }
 828
 829     if ((dabrx & ~0xfULL) != 0 || (dabrx & H_DABRX_HYPERVISOR) != 0
 830         || (dabrx & (H_DABRX_KERNEL | H_DABRX_USER)) == 0) {
 831         return H_PARAMETER;
 832     }
 833
 834     cpu_synchronize_state(CPU(cpu));
 835     cpu->env.spr[SPR_DABRX] = dabrx;
 836     cpu->env.spr[SPR_DABR] = args[0];
 837
 838     return H_SUCCESS;
 839 }
 840
 841 static target_ulong h_page_init(PowerPCCPU *cpu, sPAPRMachineState *spapr,
 842                                 target_ulong opcode, target_ulong *args)
 843 {
 844     target_ulong flags = args[0];
 845     hwaddr dst = args[1];
 846     hwaddr src = args[2];
 847     hwaddr len = TARGET_PAGE_SIZE;
 848     uint8_t *pdst, *psrc;
 849     target_long ret = H_SUCCESS;
 850
 851     if (flags & ~(H_ICACHE_SYNCHRONIZE | H_ICACHE_INVALIDATE
 852                   | H_COPY_PAGE | H_ZERO_PAGE)) {
 853         qemu_log_mask(LOG_UNIMP, "h_page_init: Bad flags (" TARGET_FMT_lx "\n",
 854                       flags);
 855         return H_PARAMETER;
 856     }
 857
 858     /* Map-in destination */
 859     if (!is_ram_address(spapr, dst) || (dst & ~TARGET_PAGE_MASK) != 0) {
 860         return H_PARAMETER;
 861     }
 862     pdst = cpu_physical_memory_map(dst, &len, 1);
 863     if (!pdst || len != TARGET_PAGE_SIZE) {
 864         return H_PARAMETER;
 865     }
 866
 867     if (flags & H_COPY_PAGE) {
 868         /* Map-in source, copy to destination, and unmap source again */
 869         if (!is_ram_address(spapr, src) || (src & ~TARGET_PAGE_MASK) != 0) {
 870             ret = H_PARAMETER;
 871             goto unmap_out;
 872         }
 873         psrc = cpu_physical_memory_map(src, &len, 0);
 874         if (!psrc || len != TARGET_PAGE_SIZE) {
 875             ret = H_PARAMETER;
 876             goto unmap_out;
 877         }
 878         memcpy(pdst, psrc, len);
 879         cpu_physical_memory_unmap(psrc, len, 0, len);
 880     } else if (flags & H_ZERO_PAGE) {
 881         memset(pdst, 0, len);          /* Just clear the destination page */
 882     }
 883
 884     if (kvm_enabled() && (flags & H_ICACHE_SYNCHRONIZE) != 0) {
 885         kvmppc_dcbst_range(cpu, pdst, len);
 886     }
 887     if (flags & (H_ICACHE_SYNCHRONIZE | H_ICACHE_INVALIDATE)) {
 888         if (kvm_enabled()) {
 889             kvmppc_icbi_range(cpu, pdst, len);
 890         } else {
 891             tb_flush(CPU(cpu));
 892         }
 893     }
 894
 895 unmap_out:
 896     cpu_physical_memory_unmap(pdst, TARGET_PAGE_SIZE, 1, len);
 897     return ret;
 898 }
 899
 900 #define FLAGS_REGISTER_VPA         0x0000200000000000ULL
 901 #define FLAGS_REGISTER_DTL         0x0000400000000000ULL
 902 #define FLAGS_REGISTER_SLBSHADOW   0x0000600000000000ULL
 903 #define FLAGS_DEREGISTER_VPA       0x0000a00000000000ULL
 904 #define FLAGS_DEREGISTER_DTL       0x0000c00000000000ULL
 905 #define FLAGS_DEREGISTER_SLBSHADOW 0x0000e00000000000ULL
 906
 907 #define VPA_MIN_SIZE           640
 908 #define VPA_SIZE_OFFSET        0x4
 909 #define VPA_SHARED_PROC_OFFSET 0x9
 910 #define VPA_SHARED_PROC_VAL    0x2
 911
 912 static target_ulong register_vpa(PowerPCCPU *cpu, target_ulong vpa)
 913 {
 914     CPUState *cs = CPU(cpu);
 915     CPUPPCState *env = &cpu->env;
 916     sPAPRCPUState *spapr_cpu = spapr_cpu_state(cpu);
 917     uint16_t size;
 918     uint8_t tmp;
 919
 920     if (vpa == 0) {
 921         hcall_dprintf("Can't cope with registering a VPA at logical 0\n");
 922         return H_HARDWARE;
 923     }
 924
 925     if (vpa % env->dcache_line_size) {
 926         return H_PARAMETER;
 927     }
 928     /* FIXME: bounds check the address */
 929
 930     size = lduw_be_phys(cs->as, vpa + 0x4);
 931
 932     if (size < VPA_MIN_SIZE) {
 933         return H_PARAMETER;
 934     }
 935
 936     /* VPA is not allowed to cross a page boundary */
 937     if ((vpa / 4096) != ((vpa + size - 1) / 4096)) {
 938         return H_PARAMETER;
 939     }
 940
 941     spapr_cpu->vpa_addr = vpa;
 942
 943     tmp = ldub_phys(cs->as, spapr_cpu->vpa_addr + VPA_SHARED_PROC_OFFSET);
 944     tmp |= VPA_SHARED_PROC_VAL;
 945     stb_phys(cs->as, spapr_cpu->vpa_addr + VPA_SHARED_PROC_OFFSET, tmp);
 946
 947     return H_SUCCESS;
 948 }
 949
 950 static target_ulong deregister_vpa(PowerPCCPU *cpu, target_ulong vpa)
 951 {
 952     sPAPRCPUState *spapr_cpu = spapr_cpu_state(cpu);
 953
 954     if (spapr_cpu->slb_shadow_addr) {
 955         return H_RESOURCE;
 956     }
 957
 958     if (spapr_cpu->dtl_addr) {
 959         return H_RESOURCE;
 960     }
 961
 962     spapr_cpu->vpa_addr = 0;
 963     return H_SUCCESS;
 964 }
 965
 966 static target_ulong register_slb_shadow(PowerPCCPU *cpu, target_ulong addr)
 967 {
 968     sPAPRCPUState *spapr_cpu = spapr_cpu_state(cpu);
 969     uint32_t size;
 970
 971     if (addr == 0) {
 972         hcall_dprintf("Can't cope with SLB shadow at logical 0\n");
 973         return H_HARDWARE;
 974     }
 975
 976     size = ldl_be_phys(CPU(cpu)->as, addr + 0x4);
 977     if (size < 0x8) {
 978         return H_PARAMETER;
 979     }
 980
 981     if ((addr / 4096) != ((addr + size - 1) / 4096)) {
 982         return H_PARAMETER;
 983     }
 984
 985     if (!spapr_cpu->vpa_addr) {
 986         return H_RESOURCE;
 987     }
 988
 989     spapr_cpu->slb_shadow_addr = addr;
 990     spapr_cpu->slb_shadow_size = size;
 991
 992     return H_SUCCESS;
 993 }
 994
 995 static target_ulong deregister_slb_shadow(PowerPCCPU *cpu, target_ulong addr)
 996 {
 997     sPAPRCPUState *spapr_cpu = spapr_cpu_state(cpu);
 998
 999     spapr_cpu->slb_shadow_addr = 0;
1000     spapr_cpu->slb_shadow_size = 0;
1001     return H_SUCCESS;
1002 }
1003
1004 static target_ulong register_dtl(PowerPCCPU *cpu, target_ulong addr)
1005 {
1006     sPAPRCPUState *spapr_cpu = spapr_cpu_state(cpu);
1007     uint32_t size;
1008
1009     if (addr == 0) {
1010         hcall_dprintf("Can't cope with DTL at logical 0\n");
1011         return H_HARDWARE;
1012     }
1013
1014     size = ldl_be_phys(CPU(cpu)->as, addr + 0x4);
1015
1016     if (size < 48) {
1017         return H_PARAMETER;
1018     }
1019
1020     if (!spapr_cpu->vpa_addr) {
1021         return H_RESOURCE;
1022     }
1023
1024     spapr_cpu->dtl_addr = addr;
1025     spapr_cpu->dtl_size = size;
1026
1027     return H_SUCCESS;
1028 }
1029
1030 static target_ulong deregister_dtl(PowerPCCPU *cpu, target_ulong addr)
1031 {
1032     sPAPRCPUState *spapr_cpu = spapr_cpu_state(cpu);
1033
1034     spapr_cpu->dtl_addr = 0;
1035     spapr_cpu->dtl_size = 0;
1036
1037     return H_SUCCESS;
1038 }
1039
1040 static target_ulong h_register_vpa(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1041                                    target_ulong opcode, target_ulong *args)
1042 {
1043     target_ulong flags = args[0];
1044     target_ulong procno = args[1];
1045     target_ulong vpa = args[2];
1046     target_ulong ret = H_PARAMETER;
1047     PowerPCCPU *tcpu;
1048
1049     tcpu = spapr_find_cpu(procno);
1050     if (!tcpu) {
1051         return H_PARAMETER;
1052     }
1053
1054     switch (flags) {
1055     case FLAGS_REGISTER_VPA:
1056         ret = register_vpa(tcpu, vpa);
1057         break;
1058
1059     case FLAGS_DEREGISTER_VPA:
1060         ret = deregister_vpa(tcpu, vpa);
1061         break;
1062
1063     case FLAGS_REGISTER_SLBSHADOW:
1064         ret = register_slb_shadow(tcpu, vpa);
1065         break;
1066
1067     case FLAGS_DEREGISTER_SLBSHADOW:
1068         ret = deregister_slb_shadow(tcpu, vpa);
1069         break;
1070
1071     case FLAGS_REGISTER_DTL:
1072         ret = register_dtl(tcpu, vpa);
1073         break;
1074
1075     case FLAGS_DEREGISTER_DTL:
1076         ret = deregister_dtl(tcpu, vpa);
1077         break;
1078     }
1079
1080     return ret;
1081 }
1082
1083 static target_ulong h_cede(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1084                            target_ulong opcode, target_ulong *args)
1085 {
1086     CPUPPCState *env = &cpu->env;
1087     CPUState *cs = CPU(cpu);
1088
1089     env->msr |= (1ULL << MSR_EE);
1090     hreg_compute_hflags(env);
1091     if (!cpu_has_work(cs)) {
1092         cs->halted = 1;
1093         cs->exception_index = EXCP_HLT;
1094         cs->exit_request = 1;
1095     }
1096     return H_SUCCESS;
1097 }
1098
1099 static target_ulong h_rtas(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1100                            target_ulong opcode, target_ulong *args)
1101 {
1102     target_ulong rtas_r3 = args[0];
1103     uint32_t token = rtas_ld(rtas_r3, 0);
1104     uint32_t nargs = rtas_ld(rtas_r3, 1);
1105     uint32_t nret = rtas_ld(rtas_r3, 2);
1106
1107     return spapr_rtas_call(cpu, spapr, token, nargs, rtas_r3 + 12,
1108                            nret, rtas_r3 + 12 + 4*nargs);
1109 }
1110
1111 static target_ulong h_logical_load(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1112                                    target_ulong opcode, target_ulong *args)
1113 {
1114     CPUState *cs = CPU(cpu);
1115     target_ulong size = args[0];
1116     target_ulong addr = args[1];
1117
1118     switch (size) {
1119     case 1:
1120         args[0] = ldub_phys(cs->as, addr);
1121         return H_SUCCESS;
1122     case 2:
1123         args[0] = lduw_phys(cs->as, addr);
1124         return H_SUCCESS;
1125     case 4:
1126         args[0] = ldl_phys(cs->as, addr);
1127         return H_SUCCESS;
1128     case 8:
1129         args[0] = ldq_phys(cs->as, addr);
1130         return H_SUCCESS;
1131     }
1132     return H_PARAMETER;
1133 }
1134
1135 static target_ulong h_logical_store(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1136                                     target_ulong opcode, target_ulong *args)
1137 {
1138     CPUState *cs = CPU(cpu);
1139
1140     target_ulong size = args[0];
1141     target_ulong addr = args[1];
1142     target_ulong val  = args[2];
1143
1144     switch (size) {
1145     case 1:
1146         stb_phys(cs->as, addr, val);
1147         return H_SUCCESS;
1148     case 2:
1149         stw_phys(cs->as, addr, val);
1150         return H_SUCCESS;
1151     case 4:
1152         stl_phys(cs->as, addr, val);
1153         return H_SUCCESS;
1154     case 8:
1155         stq_phys(cs->as, addr, val);
1156         return H_SUCCESS;
1157     }
1158     return H_PARAMETER;
1159 }
1160
1161 static target_ulong h_logical_memop(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1162                                     target_ulong opcode, target_ulong *args)
1163 {
1164     CPUState *cs = CPU(cpu);
1165
1166     target_ulong dst   = args[0]; /* Destination address */
1167     target_ulong src   = args[1]; /* Source address */
1168     target_ulong esize = args[2]; /* Element size (0=1,1=2,2=4,3=8) */
1169     target_ulong count = args[3]; /* Element count */
1170     target_ulong op    = args[4]; /* 0 = copy, 1 = invert */
1171     uint64_t tmp;
1172     unsigned int mask = (1 << esize) - 1;
1173     int step = 1 << esize;
1174
1175     if (count > 0x80000000) {
1176         return H_PARAMETER;
1177     }
1178
1179     if ((dst & mask) || (src & mask) || (op > 1)) {
1180         return H_PARAMETER;
1181     }
1182
1183     if (dst >= src && dst < (src + (count << esize))) {
1184             dst = dst + ((count - 1) << esize);
1185             src = src + ((count - 1) << esize);
1186             step = -step;
1187     }
1188
1189     while (count--) {
1190         switch (esize) {
1191         case 0:
1192             tmp = ldub_phys(cs->as, src);
1193             break;
1194         case 1:
1195             tmp = lduw_phys(cs->as, src);
1196             break;
1197         case 2:
1198             tmp = ldl_phys(cs->as, src);
1199             break;
1200         case 3:
1201             tmp = ldq_phys(cs->as, src);
1202             break;
1203         default:
1204             return H_PARAMETER;
1205         }
1206         if (op == 1) {
1207             tmp = ~tmp;
1208         }
1209         switch (esize) {
1210         case 0:
1211             stb_phys(cs->as, dst, tmp);
1212             break;
1213         case 1:
1214             stw_phys(cs->as, dst, tmp);
1215             break;
1216         case 2:
1217             stl_phys(cs->as, dst, tmp);
1218             break;
1219         case 3:
1220             stq_phys(cs->as, dst, tmp);
1221             break;
1222         }
1223         dst = dst + step;
1224         src = src + step;
1225     }
1226
1227     return H_SUCCESS;
1228 }
1229
1230 static target_ulong h_logical_icbi(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1231                                    target_ulong opcode, target_ulong *args)
1232 {
1233     /* Nothing to do on emulation, KVM will trap this in the kernel */
1234     return H_SUCCESS;
1235 }
1236
1237 static target_ulong h_logical_dcbf(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1238                                    target_ulong opcode, target_ulong *args)
1239 {
1240     /* Nothing to do on emulation, KVM will trap this in the kernel */
1241     return H_SUCCESS;
1242 }
1243
1244 static target_ulong h_set_mode_resource_le(PowerPCCPU *cpu,
1245                                            target_ulong mflags,
1246                                            target_ulong value1,
1247                                            target_ulong value2)
1248 {
1249     if (value1) {
1250         return H_P3;
1251     }
1252     if (value2) {
1253         return H_P4;
1254     }
1255
1256     switch (mflags) {
1257     case H_SET_MODE_ENDIAN_BIG:
1258         set_all_lpcrs(0, LPCR_ILE);
1259         spapr_pci_switch_vga(true);
1260         return H_SUCCESS;
1261
1262     case H_SET_MODE_ENDIAN_LITTLE:
1263         set_all_lpcrs(LPCR_ILE, LPCR_ILE);
1264         spapr_pci_switch_vga(false);
1265         return H_SUCCESS;
1266     }
1267
1268     return H_UNSUPPORTED_FLAG;
1269 }
1270
1271 static target_ulong h_set_mode_resource_addr_trans_mode(PowerPCCPU *cpu,
1272                                                         target_ulong mflags,
1273                                                         target_ulong value1,
1274                                                         target_ulong value2)
1275 {
1276     PowerPCCPUClass *pcc = POWERPC_CPU_GET_CLASS(cpu);
1277
1278     if (!(pcc->insns_flags2 & PPC2_ISA207S)) {
1279         return H_P2;
1280     }
1281     if (value1) {
1282         return H_P3;
1283     }
1284     if (value2) {
1285         return H_P4;
1286     }
1287
1288     if (mflags == AIL_RESERVED) {
1289         return H_UNSUPPORTED_FLAG;
1290     }
1291
1292     set_all_lpcrs(mflags << LPCR_AIL_SHIFT, LPCR_AIL);
1293
1294     return H_SUCCESS;
1295 }
1296
1297 static target_ulong h_set_mode(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1298                                target_ulong opcode, target_ulong *args)
1299 {
1300     target_ulong resource = args[1];
1301     target_ulong ret = H_P2;
1302
1303     switch (resource) {
1304     case H_SET_MODE_RESOURCE_LE:
1305         ret = h_set_mode_resource_le(cpu, args[0], args[2], args[3]);
1306         break;
1307     case H_SET_MODE_RESOURCE_ADDR_TRANS_MODE:
1308         ret = h_set_mode_resource_addr_trans_mode(cpu, args[0],
1309                                                   args[2], args[3]);
1310         break;
1311     }
1312
1313     return ret;
1314 }
1315
1316 static target_ulong h_clean_slb(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1317                                 target_ulong opcode, target_ulong *args)
1318 {
1319     qemu_log_mask(LOG_UNIMP, "Unimplemented SPAPR hcall 0x"TARGET_FMT_lx"%s\n",
1320                   opcode, " (H_CLEAN_SLB)");
1321     return H_FUNCTION;
1322 }
1323
1324 static target_ulong h_invalidate_pid(PowerPCCPU *cpu, sPAPRMachineState *spapr,
1325                                      target_ulong opcode, target_ulong *args)
1326 {
1327     qemu_log_mask(LOG_UNIMP, "Unimplemented SPAPR hcall 0x"TARGET_FMT_lx"%s\n",
1328                   opcode, " (H_INVALIDATE_PID)");
1329     return H_FUNCTION;
1330 }
1331
1332 static void spapr_check_setup_free_hpt(sPAPRMachineState *spapr,
1333                                        uint64_t patbe_old, uint64_t patbe_new)
1334 {
1335     /*
1336      * We have 4 Options:
1337      * HASH->HASH || RADIX->RADIX || NOTHING->RADIX : Do Nothing
1338      * HASH->RADIX                                  : Free HPT
1339      * RADIX->HASH                                  : Allocate HPT
1340      * NOTHING->HASH                                : Allocate HPT
1341      * Note: NOTHING implies the case where we said the guest could choose
1342      *       later and so assumed radix and now it's called H_REG_PROC_TBL
1343      */
1344
1345     if ((patbe_old & PATBE1_GR) == (patbe_new & PATBE1_GR)) {
1346         /* We assume RADIX, so this catches all the "Do Nothing" cases */
1347     } else if (!(patbe_old & PATBE1_GR)) {
1348         /* HASH->RADIX : Free HPT */
1349         spapr_free_hpt(spapr);
1350     } else if (!(patbe_new & PATBE1_GR)) {
1351         /* RADIX->HASH || NOTHING->HASH : Allocate HPT */
1352         spapr_setup_hpt_and_vrma(spapr);
1353     }
1354     return;
1355 }
1356
1357 #define FLAGS_MASK              0x01FULL
1358 #define FLAG_MODIFY             0x10
1359 #define FLAG_REGISTER           0x08
1360 #define FLAG_RADIX              0x04
1361 #define FLAG_HASH_PROC_TBL      0x02
1362 #define FLAG_GTSE               0x01
1363
1364 static target_ulong h_register_process_table(PowerPCCPU *cpu,
1365                                              sPAPRMachineState *spapr,
1366                                              target_ulong opcode,
1367                                              target_ulong *args)
1368 {
1369     target_ulong flags = args[0];
1370     target_ulong proc_tbl = args[1];
1371     target_ulong page_size = args[2];
1372     target_ulong table_size = args[3];
1373     uint64_t cproc;
1374
1375     if (flags & ~FLAGS_MASK) { /* Check no reserved bits are set */
1376         return H_PARAMETER;
1377     }
1378     if (flags & FLAG_MODIFY) {
1379         if (flags & FLAG_REGISTER) {
1380             if (flags & FLAG_RADIX) { /* Register new RADIX process table */
1381                 if (proc_tbl & 0xfff || proc_tbl >> 60) {
1382                     return H_P2;
1383                 } else if (page_size) {
1384                     return H_P3;
1385                 } else if (table_size > 24) {
1386                     return H_P4;
1387                 }
1388                 cproc = PATBE1_GR | proc_tbl | table_size;
1389             } else { /* Register new HPT process table */
1390                 if (flags & FLAG_HASH_PROC_TBL) { /* Hash with Segment Tables */
1391                     /* TODO - Not Supported */
1392                     /* Technically caused by flag bits => H_PARAMETER */
1393                     return H_PARAMETER;
1394                 } else { /* Hash with SLB */
1395                     if (proc_tbl >> 38) {
1396                         return H_P2;
1397                     } else if (page_size & ~0x7) {
1398                         return H_P3;
1399                     } else if (table_size > 24) {
1400                         return H_P4;
1401                     }
1402                 }
1403                 cproc = (proc_tbl << 25) | page_size << 5 | table_size;
1404             }
1405
1406         } else { /* Deregister current process table */
1407             /* Set to benign value: (current GR) | 0. This allows
1408              * deregistration in KVM to succeed even if the radix bit in flags
1409              * doesn't match the radix bit in the old PATB. */
1410             cproc = spapr->patb_entry & PATBE1_GR;
1411         }
1412     } else { /* Maintain current registration */
1413         if (!(flags & FLAG_RADIX) != !(spapr->patb_entry & PATBE1_GR)) {
1414             /* Technically caused by flag bits => H_PARAMETER */
1415             return H_PARAMETER; /* Existing Process Table Mismatch */
1416         }
1417         cproc = spapr->patb_entry;
1418     }
1419
1420     /* Check if we need to setup OR free the hpt */
1421     spapr_check_setup_free_hpt(spapr, spapr->patb_entry, cproc);
1422
1423     spapr->patb_entry = cproc; /* Save new process table */
1424
1425     /* Update the UPRT and GTSE bits in the LPCR for all cpus */
1426     set_all_lpcrs(((flags & (FLAG_RADIX | FLAG_HASH_PROC_TBL)) ? LPCR_UPRT : 0) |
1427                   ((flags & FLAG_GTSE) ? LPCR_GTSE : 0),
1428                   LPCR_UPRT | LPCR_GTSE);
1429
1430     if (kvm_enabled()) {
1431         return kvmppc_configure_v3_mmu(cpu, flags & FLAG_RADIX,
1432                                        flags & FLAG_GTSE, cproc);
1433     }
1434     return H_SUCCESS;
1435 }
1436
1437 #define H_SIGNAL_SYS_RESET_ALL         -1
1438 #define H_SIGNAL_SYS_RESET_ALLBUTSELF  -2
1439
1440 static target_ulong h_signal_sys_reset(PowerPCCPU *cpu,
1441                                        sPAPRMachineState *spapr,
1442                                        target_ulong opcode, target_ulong *args)
1443 {
1444     target_long target = args[0];
1445     CPUState *cs;
1446
1447     if (target < 0) {
1448         /* Broadcast */
1449         if (target < H_SIGNAL_SYS_RESET_ALLBUTSELF) {
1450             return H_PARAMETER;
1451         }
1452
1453         CPU_FOREACH(cs) {
1454             PowerPCCPU *c = POWERPC_CPU(cs);
1455
1456             if (target == H_SIGNAL_SYS_RESET_ALLBUTSELF) {
1457                 if (c == cpu) {
1458                     continue;
1459                 }
1460             }
1461             run_on_cpu(cs, spapr_do_system_reset_on_cpu, RUN_ON_CPU_NULL);
1462         }
1463         return H_SUCCESS;
1464
1465     } else {
1466         /* Unicast */
1467         cs = CPU(spapr_find_cpu(target));
1468         if (cs) {
1469             run_on_cpu(cs, spapr_do_system_reset_on_cpu, RUN_ON_CPU_NULL);
1470             return H_SUCCESS;
1471         }
1472         return H_PARAMETER;
1473     }
1474 }
1475
1476 static uint32_t cas_check_pvr(sPAPRMachineState *spapr, PowerPCCPU *cpu,
1477                               target_ulong *addr, bool *raw_mode_supported,
1478                               Error **errp)
1479 {
1480     bool explicit_match = false; /* Matched the CPU's real PVR */
1481     uint32_t max_compat = spapr->max_compat_pvr;
1482     uint32_t best_compat = 0;
1483     int i;
1484
1485     /*
1486      * We scan the supplied table of PVRs looking for two things
1487      *   1. Is our real CPU PVR in the list?
1488      *   2. What's the "best" listed logical PVR
1489      */
1490     for (i = 0; i < 512; ++i) {
1491         uint32_t pvr, pvr_mask;
1492
1493         pvr_mask = ldl_be_phys(&address_space_memory, *addr);
1494         pvr = ldl_be_phys(&address_space_memory, *addr + 4);
1495         *addr += 8;
1496
1497         if (~pvr_mask & pvr) {
1498             break; /* Terminator record */
1499         }
1500
1501         if ((cpu->env.spr[SPR_PVR] & pvr_mask) == (pvr & pvr_mask)) {
1502             explicit_match = true;
1503         } else {
1504             if (ppc_check_compat(cpu, pvr, best_compat, max_compat)) {
1505                 best_compat = pvr;
1506             }
1507         }
1508     }
1509
1510     if ((best_compat == 0) && (!explicit_match || max_compat)) {
1511         /* We couldn't find a suitable compatibility mode, and either
1512          * the guest doesn't support "raw" mode for this CPU, or raw
1513          * mode is disabled because a maximum compat mode is set */
1514         error_setg(errp, "Couldn't negotiate a suitable PVR during CAS");
1515         return 0;
1516     }
1517
1518     *raw_mode_supported = explicit_match;
1519
1520     /* Parsing finished */
1521     trace_spapr_cas_pvr(cpu->compat_pvr, explicit_match, best_compat);
1522
1523     return best_compat;
1524 }
1525
1526 static target_ulong h_client_architecture_support(PowerPCCPU *cpu,
1527                                                   sPAPRMachineState *spapr,
1528                                                   target_ulong opcode,
1529                                                   target_ulong *args)
1530 {
1531     /* Working address in data buffer */
1532     target_ulong addr = ppc64_phys_to_real(args[0]);
1533     target_ulong ov_table;
1534     uint32_t cas_pvr;
1535     sPAPROptionVector *ov1_guest, *ov5_guest, *ov5_cas_old, *ov5_updates;
1536     bool guest_radix;
1537     Error *local_err = NULL;
1538     bool raw_mode_supported = false;
1539
1540     cas_pvr = cas_check_pvr(spapr, cpu, &addr, &raw_mode_supported, &local_err);
1541     if (local_err) {
1542         error_report_err(local_err);
1543         return H_HARDWARE;
1544     }
1545
1546     /* Update CPUs */
1547     if (cpu->compat_pvr != cas_pvr) {
1548         ppc_set_compat_all(cas_pvr, &local_err);
1549         if (local_err) {
1550             /* We fail to set compat mode (likely because running with KVM PR),
1551              * but maybe we can fallback to raw mode if the guest supports it.
1552              */
1553             if (!raw_mode_supported) {
1554                 error_report_err(local_err);
1555                 return H_HARDWARE;
1556             }
1557             error_free(local_err);
1558             local_err = NULL;
1559         }
1560     }
1561
1562     /* For the future use: here @ov_table points to the first option vector */
1563     ov_table = addr;
1564
1565     ov1_guest = spapr_ovec_parse_vector(ov_table, 1);
1566     ov5_guest = spapr_ovec_parse_vector(ov_table, 5);
1567     if (spapr_ovec_test(ov5_guest, OV5_MMU_BOTH)) {
1568         error_report("guest requested hash and radix MMU, which is invalid.");
1569         exit(EXIT_FAILURE);
1570     }
1571     /* The radix/hash bit in byte 24 requires special handling: */
1572     guest_radix = spapr_ovec_test(ov5_guest, OV5_MMU_RADIX_300);
1573     spapr_ovec_clear(ov5_guest, OV5_MMU_RADIX_300);
1574
1575     /*
1576      * HPT resizing is a bit of a special case, because when enabled
1577      * we assume an HPT guest will support it until it says it
1578      * doesn't, instead of assuming it won't support it until it says
1579      * it does.  Strictly speaking that approach could break for
1580      * guests which don't make a CAS call, but those are so old we
1581      * don't care about them.  Without that assumption we'd have to
1582      * make at least a temporary allocation of an HPT sized for max
1583      * memory, which could be impossibly difficult under KVM HV if
1584      * maxram is large.
1585      */
1586     if (!guest_radix && !spapr_ovec_test(ov5_guest, OV5_HPT_RESIZE)) {
1587         int maxshift = spapr_hpt_shift_for_ramsize(MACHINE(spapr)->maxram_size);
1588
1589         if (spapr->resize_hpt == SPAPR_RESIZE_HPT_REQUIRED) {
1590             error_report(
1591                 "h_client_architecture_support: Guest doesn't support HPT resizing, but resize-hpt=required");
1592             exit(1);
1593         }
1594
1595         if (spapr->htab_shift < maxshift) {
1596             /* Guest doesn't know about HPT resizing, so we
1597              * pre-emptively resize for the maximum permitted RAM.  At
1598              * the point this is called, nothing should have been
1599              * entered into the existing HPT */
1600             spapr_reallocate_hpt(spapr, maxshift, &error_fatal);
1601             push_sregs_to_kvm_pr(spapr);
1602         }
1603     }
1604
1605     /* NOTE: there are actually a number of ov5 bits where input from the
1606      * guest is always zero, and the platform/QEMU enables them independently
1607      * of guest input. To model these properly we'd want some sort of mask,
1608      * but since they only currently apply to memory migration as defined
1609      * by LoPAPR 1.1, 14.5.4.8, which QEMU doesn't implement, we don't need
1610      * to worry about this for now.
1611      */
1612     ov5_cas_old = spapr_ovec_clone(spapr->ov5_cas);
1613
1614     /* also clear the radix/hash bit from the current ov5_cas bits to
1615      * be in sync with the newly ov5 bits. Else the radix bit will be
1616      * seen as being removed and this will generate a reset loop
1617      */
1618     spapr_ovec_clear(ov5_cas_old, OV5_MMU_RADIX_300);
1619
1620     /* full range of negotiated ov5 capabilities */
1621     spapr_ovec_intersect(spapr->ov5_cas, spapr->ov5, ov5_guest);
1622     spapr_ovec_cleanup(ov5_guest);
1623     /* capabilities that have been added since CAS-generated guest reset.
1624      * if capabilities have since been removed, generate another reset
1625      */
1626     ov5_updates = spapr_ovec_new();
1627     spapr->cas_reboot = spapr_ovec_diff(ov5_updates,
1628                                         ov5_cas_old, spapr->ov5_cas);
1629     /* Now that processing is finished, set the radix/hash bit for the
1630      * guest if it requested a valid mode; otherwise terminate the boot. */
1631     if (guest_radix) {
1632         if (kvm_enabled() && !kvmppc_has_cap_mmu_radix()) {
1633             error_report("Guest requested unavailable MMU mode (radix).");
1634             exit(EXIT_FAILURE);
1635         }
1636         spapr_ovec_set(spapr->ov5_cas, OV5_MMU_RADIX_300);
1637     } else {
1638         if (kvm_enabled() && kvmppc_has_cap_mmu_radix()
1639             && !kvmppc_has_cap_mmu_hash_v3()) {
1640             error_report("Guest requested unavailable MMU mode (hash).");
1641             exit(EXIT_FAILURE);
1642         }
1643     }
1644     spapr->cas_legacy_guest_workaround = !spapr_ovec_test(ov1_guest,
1645                                                           OV1_PPC_3_00);
1646     if (!spapr->cas_reboot) {
1647         /* If spapr_machine_reset() did not set up a HPT but one is necessary
1648          * (because the guest isn't going to use radix) then set it up here. */
1649         if ((spapr->patb_entry & PATBE1_GR) && !guest_radix) {
1650             /* legacy hash or new hash: */
1651             spapr_setup_hpt_and_vrma(spapr);
1652         }
1653         spapr->cas_reboot =
1654             (spapr_h_cas_compose_response(spapr, args[1], args[2],
1655                                           ov5_updates) != 0);
1656     }
1657     spapr_ovec_cleanup(ov5_updates);
1658
1659     if (spapr->cas_reboot) {
1660         qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
1661     }
1662
1663     return H_SUCCESS;
1664 }
1665
1666 static target_ulong h_get_cpu_characteristics(PowerPCCPU *cpu,
1667                                               sPAPRMachineState *spapr,
1668                                               target_ulong opcode,
1669                                               target_ulong *args)
1670 {
1671     uint64_t characteristics = H_CPU_CHAR_HON_BRANCH_HINTS &
1672                                ~H_CPU_CHAR_THR_RECONF_TRIG;
1673     uint64_t behaviour = H_CPU_BEHAV_FAVOUR_SECURITY;
1674     uint8_t safe_cache = spapr_get_cap(spapr, SPAPR_CAP_CFPC);
1675     uint8_t safe_bounds_check = spapr_get_cap(spapr, SPAPR_CAP_SBBC);
1676     uint8_t safe_indirect_branch = spapr_get_cap(spapr, SPAPR_CAP_IBS);
1677
1678     switch (safe_cache) {
1679     case SPAPR_CAP_WORKAROUND:
1680         characteristics |= H_CPU_CHAR_L1D_FLUSH_ORI30;
1681         characteristics |= H_CPU_CHAR_L1D_FLUSH_TRIG2;
1682         characteristics |= H_CPU_CHAR_L1D_THREAD_PRIV;
1683         behaviour |= H_CPU_BEHAV_L1D_FLUSH_PR;
1684         break;
1685     case SPAPR_CAP_FIXED:
1686         break;
1687     default: /* broken */
1688         assert(safe_cache == SPAPR_CAP_BROKEN);
1689         behaviour |= H_CPU_BEHAV_L1D_FLUSH_PR;
1690         break;
1691     }
1692
1693     switch (safe_bounds_check) {
1694     case SPAPR_CAP_WORKAROUND:
1695         characteristics |= H_CPU_CHAR_SPEC_BAR_ORI31;
1696         behaviour |= H_CPU_BEHAV_BNDS_CHK_SPEC_BAR;
1697         break;
1698     case SPAPR_CAP_FIXED:
1699         break;
1700     default: /* broken */
1701         assert(safe_bounds_check == SPAPR_CAP_BROKEN);
1702         behaviour |= H_CPU_BEHAV_BNDS_CHK_SPEC_BAR;
1703         break;
1704     }
1705
1706     switch (safe_indirect_branch) {
1707     case SPAPR_CAP_FIXED_CCD:
1708         characteristics |= H_CPU_CHAR_CACHE_COUNT_DIS;
1709         break;
1710     case SPAPR_CAP_FIXED_IBS:
1711         characteristics |= H_CPU_CHAR_BCCTRL_SERIALISED;
1712         break;
1713     default: /* broken */
1714         assert(safe_indirect_branch == SPAPR_CAP_BROKEN);
1715         break;
1716     }
1717
1718     args[0] = characteristics;
1719     args[1] = behaviour;
1720
1721     return H_SUCCESS;
1722 }
1723
1724 static spapr_hcall_fn papr_hypercall_table[(MAX_HCALL_OPCODE / 4) + 1];
1725 static spapr_hcall_fn kvmppc_hypercall_table[KVMPPC_HCALL_MAX - KVMPPC_HCALL_BASE + 1];
1726
1727 void spapr_register_hypercall(target_ulong opcode, spapr_hcall_fn fn)
1728 {
1729     spapr_hcall_fn *slot;
1730
1731     if (opcode <= MAX_HCALL_OPCODE) {
1732         assert((opcode & 0x3) == 0);
1733
1734         slot = &papr_hypercall_table[opcode / 4];
1735     } else {
1736         assert((opcode >= KVMPPC_HCALL_BASE) && (opcode <= KVMPPC_HCALL_MAX));
1737
1738         slot = &kvmppc_hypercall_table[opcode - KVMPPC_HCALL_BASE];
1739     }
1740
1741     assert(!(*slot));
1742     *slot = fn;
1743 }
1744
1745 target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
1746                              target_ulong *args)
1747 {
1748     sPAPRMachineState *spapr = SPAPR_MACHINE(qdev_get_machine());
1749
1750     if ((opcode <= MAX_HCALL_OPCODE)
1751         && ((opcode & 0x3) == 0)) {
1752         spapr_hcall_fn fn = papr_hypercall_table[opcode / 4];
1753
1754         if (fn) {
1755             return fn(cpu, spapr, opcode, args);
1756         }
1757     } else if ((opcode >= KVMPPC_HCALL_BASE) &&
1758                (opcode <= KVMPPC_HCALL_MAX)) {
1759         spapr_hcall_fn fn = kvmppc_hypercall_table[opcode - KVMPPC_HCALL_BASE];
1760
1761         if (fn) {
1762             return fn(cpu, spapr, opcode, args);
1763         }
1764     }
1765
1766     qemu_log_mask(LOG_UNIMP, "Unimplemented SPAPR hcall 0x" TARGET_FMT_lx "\n",
1767                   opcode);
1768     return H_FUNCTION;
1769 }
1770
1771 static void hypercall_register_types(void)
1772 {
1773     /* hcall-pft */
1774     spapr_register_hypercall(H_ENTER, h_enter);
1775     spapr_register_hypercall(H_REMOVE, h_remove);
1776     spapr_register_hypercall(H_PROTECT, h_protect);
1777     spapr_register_hypercall(H_READ, h_read);
1778
1779     /* hcall-bulk */
1780     spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
1781
1782     /* hcall-hpt-resize */
1783     spapr_register_hypercall(H_RESIZE_HPT_PREPARE, h_resize_hpt_prepare);
1784     spapr_register_hypercall(H_RESIZE_HPT_COMMIT, h_resize_hpt_commit);
1785
1786     /* hcall-splpar */
1787     spapr_register_hypercall(H_REGISTER_VPA, h_register_vpa);
1788     spapr_register_hypercall(H_CEDE, h_cede);
1789     spapr_register_hypercall(H_SIGNAL_SYS_RESET, h_signal_sys_reset);
1790
1791     /* processor register resource access h-calls */
1792     spapr_register_hypercall(H_SET_SPRG0, h_set_sprg0);
1793     spapr_register_hypercall(H_SET_DABR, h_set_dabr);
1794     spapr_register_hypercall(H_SET_XDABR, h_set_xdabr);
1795     spapr_register_hypercall(H_PAGE_INIT, h_page_init);
1796     spapr_register_hypercall(H_SET_MODE, h_set_mode);
1797
1798     /* In Memory Table MMU h-calls */
1799     spapr_register_hypercall(H_CLEAN_SLB, h_clean_slb);
1800     spapr_register_hypercall(H_INVALIDATE_PID, h_invalidate_pid);
1801     spapr_register_hypercall(H_REGISTER_PROC_TBL, h_register_process_table);
1802
1803     /* hcall-get-cpu-characteristics */
1804     spapr_register_hypercall(H_GET_CPU_CHARACTERISTICS,
1805                              h_get_cpu_characteristics);
1806
1807     /* "debugger" hcalls (also used by SLOF). Note: We do -not- differenciate
1808      * here between the "CI" and the "CACHE" variants, they will use whatever
1809      * mapping attributes qemu is using. When using KVM, the kernel will
1810      * enforce the attributes more strongly
1811      */
1812     spapr_register_hypercall(H_LOGICAL_CI_LOAD, h_logical_load);
1813     spapr_register_hypercall(H_LOGICAL_CI_STORE, h_logical_store);
1814     spapr_register_hypercall(H_LOGICAL_CACHE_LOAD, h_logical_load);
1815     spapr_register_hypercall(H_LOGICAL_CACHE_STORE, h_logical_store);
1816     spapr_register_hypercall(H_LOGICAL_ICBI, h_logical_icbi);
1817     spapr_register_hypercall(H_LOGICAL_DCBF, h_logical_dcbf);
1818     spapr_register_hypercall(KVMPPC_H_LOGICAL_MEMOP, h_logical_memop);
1819
1820     /* qemu/KVM-PPC specific hcalls */
1821     spapr_register_hypercall(KVMPPC_H_RTAS, h_rtas);
1822
1823     /* ibm,client-architecture-support support */
1824     spapr_register_hypercall(KVMPPC_H_CAS, h_client_architecture_support);
1825 }
1826
1827 type_init(hypercall_register_types)