hw/ppc/spapr_softmmu.c

   1 #include "qemu/osdep.h"
   2 #include "qemu/cutils.h"
   3 #include "cpu.h"
   4 #include "helper_regs.h"
   5 #include "hw/ppc/spapr.h"
   6 #include "mmu-hash64.h"
   7 #include "mmu-book3s-v3.h"
   8
   9 static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
  10 {
  11     /*
  12      * hash value/pteg group index is normalized by HPT mask
  13      */
  14     if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
  15         return false;
  16     }
  17     return true;
  18 }
  19
  20 static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
  21                             target_ulong opcode, target_ulong *args)
  22 {
  23     target_ulong flags = args[0];
  24     target_ulong ptex = args[1];
  25     target_ulong pteh = args[2];
  26     target_ulong ptel = args[3];
  27     unsigned apshift;
  28     target_ulong raddr;
  29     target_ulong slot;
  30     const ppc_hash_pte64_t *hptes;
  31
  32     apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
  33     if (!apshift) {
  34         /* Bad page size encoding */
  35         return H_PARAMETER;
  36     }
  37
  38     raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
  39
  40     if (is_ram_address(spapr, raddr)) {
  41         /* Regular RAM - should have WIMG=0010 */
  42         if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
  43             return H_PARAMETER;
  44         }
  45     } else {
  46         target_ulong wimg_flags;
  47         /* Looks like an IO address */
  48         /* FIXME: What WIMG combinations could be sensible for IO?
  49          * For now we allow WIMG=010x, but are there others? */
  50         /* FIXME: Should we check against registered IO addresses? */
  51         wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
  52
  53         if (wimg_flags != HPTE64_R_I &&
  54             wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
  55             return H_PARAMETER;
  56         }
  57     }
  58
  59     pteh &= ~0x60ULL;
  60
  61     if (!valid_ptex(cpu, ptex)) {
  62         return H_PARAMETER;
  63     }
  64
  65     slot = ptex & 7ULL;
  66     ptex = ptex & ~7ULL;
  67
  68     if (likely((flags & H_EXACT) == 0)) {
  69         hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
  70         for (slot = 0; slot < 8; slot++) {
  71             if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
  72                 break;
  73             }
  74         }
  75         ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
  76         if (slot == 8) {
  77             return H_PTEG_FULL;
  78         }
  79     } else {
  80         hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
  81         if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
  82             ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
  83             return H_PTEG_FULL;
  84         }
  85         ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
  86     }
  87
  88     spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
  89
  90     args[0] = ptex + slot;
  91     return H_SUCCESS;
  92 }
  93
  94 typedef enum {
  95     REMOVE_SUCCESS = 0,
  96     REMOVE_NOT_FOUND = 1,
  97     REMOVE_PARM = 2,
  98     REMOVE_HW = 3,
  99 } RemoveResult;
 100
 101 static RemoveResult remove_hpte(PowerPCCPU *cpu
 102                                 , target_ulong ptex,
 103                                 target_ulong avpn,
 104                                 target_ulong flags,
 105                                 target_ulong *vp, target_ulong *rp)
 106 {
 107     const ppc_hash_pte64_t *hptes;
 108     target_ulong v, r;
 109
 110     if (!valid_ptex(cpu, ptex)) {
 111         return REMOVE_PARM;
 112     }
 113
 114     hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
 115     v = ppc_hash64_hpte0(cpu, hptes, 0);
 116     r = ppc_hash64_hpte1(cpu, hptes, 0);
 117     ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
 118
 119     if ((v & HPTE64_V_VALID) == 0 ||
 120         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
 121         ((flags & H_ANDCOND) && (v & avpn) != 0)) {
 122         return REMOVE_NOT_FOUND;
 123     }
 124     *vp = v;
 125     *rp = r;
 126     spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
 127     ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
 128     return REMOVE_SUCCESS;
 129 }
 130
 131 static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
 132                              target_ulong opcode, target_ulong *args)
 133 {
 134     CPUPPCState *env = &cpu->env;
 135     target_ulong flags = args[0];
 136     target_ulong ptex = args[1];
 137     target_ulong avpn = args[2];
 138     RemoveResult ret;
 139
 140     ret = remove_hpte(cpu, ptex, avpn, flags,
 141                       &args[0], &args[1]);
 142
 143     switch (ret) {
 144     case REMOVE_SUCCESS:
 145         check_tlb_flush(env, true);
 146         return H_SUCCESS;
 147
 148     case REMOVE_NOT_FOUND:
 149         return H_NOT_FOUND;
 150
 151     case REMOVE_PARM:
 152         return H_PARAMETER;
 153
 154     case REMOVE_HW:
 155         return H_HARDWARE;
 156     }
 157
 158     g_assert_not_reached();
 159 }
 160
 161 #define H_BULK_REMOVE_TYPE             0xc000000000000000ULL
 162 #define   H_BULK_REMOVE_REQUEST        0x4000000000000000ULL
 163 #define   H_BULK_REMOVE_RESPONSE       0x8000000000000000ULL
 164 #define   H_BULK_REMOVE_END            0xc000000000000000ULL
 165 #define H_BULK_REMOVE_CODE             0x3000000000000000ULL
 166 #define   H_BULK_REMOVE_SUCCESS        0x0000000000000000ULL
 167 #define   H_BULK_REMOVE_NOT_FOUND      0x1000000000000000ULL
 168 #define   H_BULK_REMOVE_PARM           0x2000000000000000ULL
 169 #define   H_BULK_REMOVE_HW             0x3000000000000000ULL
 170 #define H_BULK_REMOVE_RC               0x0c00000000000000ULL
 171 #define H_BULK_REMOVE_FLAGS            0x0300000000000000ULL
 172 #define   H_BULK_REMOVE_ABSOLUTE       0x0000000000000000ULL
 173 #define   H_BULK_REMOVE_ANDCOND        0x0100000000000000ULL
 174 #define   H_BULK_REMOVE_AVPN           0x0200000000000000ULL
 175 #define H_BULK_REMOVE_PTEX             0x00ffffffffffffffULL
 176
 177 #define H_BULK_REMOVE_MAX_BATCH        4
 178
 179 static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
 180                                   target_ulong opcode, target_ulong *args)
 181 {
 182     CPUPPCState *env = &cpu->env;
 183     int i;
 184     target_ulong rc = H_SUCCESS;
 185
 186     for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
 187         target_ulong *tsh = &args[i*2];
 188         target_ulong tsl = args[i*2 + 1];
 189         target_ulong v, r, ret;
 190
 191         if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
 192             break;
 193         } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
 194             return H_PARAMETER;
 195         }
 196
 197         *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
 198         *tsh |= H_BULK_REMOVE_RESPONSE;
 199
 200         if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
 201             *tsh |= H_BULK_REMOVE_PARM;
 202             return H_PARAMETER;
 203         }
 204
 205         ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
 206                           (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
 207                           &v, &r);
 208
 209         *tsh |= ret << 60;
 210
 211         switch (ret) {
 212         case REMOVE_SUCCESS:
 213             *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
 214             break;
 215
 216         case REMOVE_PARM:
 217             rc = H_PARAMETER;
 218             goto exit;
 219
 220         case REMOVE_HW:
 221             rc = H_HARDWARE;
 222             goto exit;
 223         }
 224     }
 225  exit:
 226     check_tlb_flush(env, true);
 227
 228     return rc;
 229 }
 230
 231 static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
 232                               target_ulong opcode, target_ulong *args)
 233 {
 234     CPUPPCState *env = &cpu->env;
 235     target_ulong flags = args[0];
 236     target_ulong ptex = args[1];
 237     target_ulong avpn = args[2];
 238     const ppc_hash_pte64_t *hptes;
 239     target_ulong v, r;
 240
 241     if (!valid_ptex(cpu, ptex)) {
 242         return H_PARAMETER;
 243     }
 244
 245     hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
 246     v = ppc_hash64_hpte0(cpu, hptes, 0);
 247     r = ppc_hash64_hpte1(cpu, hptes, 0);
 248     ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
 249
 250     if ((v & HPTE64_V_VALID) == 0 ||
 251         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
 252         return H_NOT_FOUND;
 253     }
 254
 255     r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
 256            HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
 257     r |= (flags << 55) & HPTE64_R_PP0;
 258     r |= (flags << 48) & HPTE64_R_KEY_HI;
 259     r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
 260     spapr_store_hpte(cpu, ptex,
 261                      (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
 262     ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
 263     /* Flush the tlb */
 264     check_tlb_flush(env, true);
 265     /* Don't need a memory barrier, due to qemu's global lock */
 266     spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
 267     return H_SUCCESS;
 268 }
 269
 270 static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
 271                            target_ulong opcode, target_ulong *args)
 272 {
 273     target_ulong flags = args[0];
 274     target_ulong ptex = args[1];
 275     int i, ridx, n_entries = 1;
 276     const ppc_hash_pte64_t *hptes;
 277
 278     if (!valid_ptex(cpu, ptex)) {
 279         return H_PARAMETER;
 280     }
 281
 282     if (flags & H_READ_4) {
 283         /* Clear the two low order bits */
 284         ptex &= ~(3ULL);
 285         n_entries = 4;
 286     }
 287
 288     hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
 289     for (i = 0, ridx = 0; i < n_entries; i++) {
 290         args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
 291         args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
 292     }
 293     ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
 294
 295     return H_SUCCESS;
 296 }
 297
 298 struct SpaprPendingHpt {
 299     /* These fields are read-only after initialization */
 300     int shift;
 301     QemuThread thread;
 302
 303     /* These fields are protected by the BQL */
 304     bool complete;
 305
 306     /* These fields are private to the preparation thread if
 307      * !complete, otherwise protected by the BQL */
 308     int ret;
 309     void *hpt;
 310 };
 311
 312 static void free_pending_hpt(SpaprPendingHpt *pending)
 313 {
 314     if (pending->hpt) {
 315         qemu_vfree(pending->hpt);
 316     }
 317
 318     g_free(pending);
 319 }
 320
 321 static void *hpt_prepare_thread(void *opaque)
 322 {
 323     SpaprPendingHpt *pending = opaque;
 324     size_t size = 1ULL << pending->shift;
 325
 326     pending->hpt = qemu_try_memalign(size, size);
 327     if (pending->hpt) {
 328         memset(pending->hpt, 0, size);
 329         pending->ret = H_SUCCESS;
 330     } else {
 331         pending->ret = H_NO_MEM;
 332     }
 333
 334     qemu_mutex_lock_iothread();
 335
 336     if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
 337         /* Ready to go */
 338         pending->complete = true;
 339     } else {
 340         /* We've been cancelled, clean ourselves up */
 341         free_pending_hpt(pending);
 342     }
 343
 344     qemu_mutex_unlock_iothread();
 345     return NULL;
 346 }
 347
 348 /* Must be called with BQL held */
 349 static void cancel_hpt_prepare(SpaprMachineState *spapr)
 350 {
 351     SpaprPendingHpt *pending = spapr->pending_hpt;
 352
 353     /* Let the thread know it's cancelled */
 354     spapr->pending_hpt = NULL;
 355
 356     if (!pending) {
 357         /* Nothing to do */
 358         return;
 359     }
 360
 361     if (!pending->complete) {
 362         /* thread will clean itself up */
 363         return;
 364     }
 365
 366     free_pending_hpt(pending);
 367 }
 368
 369 target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
 370                                          SpaprMachineState *spapr,
 371                                          target_ulong shift)
 372 {
 373     SpaprPendingHpt *pending = spapr->pending_hpt;
 374
 375     if (pending) {
 376         /* something already in progress */
 377         if (pending->shift == shift) {
 378             /* and it's suitable */
 379             if (pending->complete) {
 380                 return pending->ret;
 381             } else {
 382                 return H_LONG_BUSY_ORDER_100_MSEC;
 383             }
 384         }
 385
 386         /* not suitable, cancel and replace */
 387         cancel_hpt_prepare(spapr);
 388     }
 389
 390     if (!shift) {
 391         /* nothing to do */
 392         return H_SUCCESS;
 393     }
 394
 395     /* start new prepare */
 396
 397     pending = g_new0(SpaprPendingHpt, 1);
 398     pending->shift = shift;
 399     pending->ret = H_HARDWARE;
 400
 401     qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
 402                        hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
 403
 404     spapr->pending_hpt = pending;
 405
 406     /* In theory we could estimate the time more accurately based on
 407      * the new size, but there's not much point */
 408     return H_LONG_BUSY_ORDER_100_MSEC;
 409 }
 410
 411 static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
 412 {
 413     uint8_t *addr = htab;
 414
 415     addr += pteg * HASH_PTEG_SIZE_64;
 416     addr += slot * HASH_PTE_SIZE_64;
 417     return  ldq_p(addr);
 418 }
 419
 420 static void new_hpte_store(void *htab, uint64_t pteg, int slot,
 421                            uint64_t pte0, uint64_t pte1)
 422 {
 423     uint8_t *addr = htab;
 424
 425     addr += pteg * HASH_PTEG_SIZE_64;
 426     addr += slot * HASH_PTE_SIZE_64;
 427
 428     stq_p(addr, pte0);
 429     stq_p(addr + HPTE64_DW1, pte1);
 430 }
 431
 432 static int rehash_hpte(PowerPCCPU *cpu,
 433                        const ppc_hash_pte64_t *hptes,
 434                        void *old_hpt, uint64_t oldsize,
 435                        void *new_hpt, uint64_t newsize,
 436                        uint64_t pteg, int slot)
 437 {
 438     uint64_t old_hash_mask = (oldsize >> 7) - 1;
 439     uint64_t new_hash_mask = (newsize >> 7) - 1;
 440     target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
 441     target_ulong pte1;
 442     uint64_t avpn;
 443     unsigned base_pg_shift;
 444     uint64_t hash, new_pteg, replace_pte0;
 445
 446     if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
 447         return H_SUCCESS;
 448     }
 449
 450     pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
 451
 452     base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
 453     assert(base_pg_shift); /* H_ENTER shouldn't allow a bad encoding */
 454     avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
 455
 456     if (pte0 & HPTE64_V_SECONDARY) {
 457         pteg = ~pteg;
 458     }
 459
 460     if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
 461         uint64_t offset, vsid;
 462
 463         /* We only have 28 - 23 bits of offset in avpn */
 464         offset = (avpn & 0x1f) << 23;
 465         vsid = avpn >> 5;
 466         /* We can find more bits from the pteg value */
 467         if (base_pg_shift < 23) {
 468             offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
 469         }
 470
 471         hash = vsid ^ (offset >> base_pg_shift);
 472     } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
 473         uint64_t offset, vsid;
 474
 475         /* We only have 40 - 23 bits of seg_off in avpn */
 476         offset = (avpn & 0x1ffff) << 23;
 477         vsid = avpn >> 17;
 478         if (base_pg_shift < 23) {
 479             offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
 480                 << base_pg_shift;
 481         }
 482
 483         hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
 484     } else {
 485         error_report("rehash_pte: Bad segment size in HPTE");
 486         return H_HARDWARE;
 487     }
 488
 489     new_pteg = hash & new_hash_mask;
 490     if (pte0 & HPTE64_V_SECONDARY) {
 491         assert(~pteg == (hash & old_hash_mask));
 492         new_pteg = ~new_pteg;
 493     } else {
 494         assert(pteg == (hash & old_hash_mask));
 495     }
 496     assert((oldsize != newsize) || (pteg == new_pteg));
 497     replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
 498     /*
 499      * Strictly speaking, we don't need all these tests, since we only
 500      * ever rehash bolted HPTEs.  We might in future handle non-bolted
 501      * HPTEs, though so make the logic correct for those cases as
 502      * well.
 503      */
 504     if (replace_pte0 & HPTE64_V_VALID) {
 505         assert(newsize < oldsize);
 506         if (replace_pte0 & HPTE64_V_BOLTED) {
 507             if (pte0 & HPTE64_V_BOLTED) {
 508                 /* Bolted collision, nothing we can do */
 509                 return H_PTEG_FULL;
 510             } else {
 511                 /* Discard this hpte */
 512                 return H_SUCCESS;
 513             }
 514         }
 515     }
 516
 517     new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
 518     return H_SUCCESS;
 519 }
 520
 521 static int rehash_hpt(PowerPCCPU *cpu,
 522                       void *old_hpt, uint64_t oldsize,
 523                       void *new_hpt, uint64_t newsize)
 524 {
 525     uint64_t n_ptegs = oldsize >> 7;
 526     uint64_t pteg;
 527     int slot;
 528     int rc;
 529
 530     for (pteg = 0; pteg < n_ptegs; pteg++) {
 531         hwaddr ptex = pteg * HPTES_PER_GROUP;
 532         const ppc_hash_pte64_t *hptes
 533             = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
 534
 535         if (!hptes) {
 536             return H_HARDWARE;
 537         }
 538
 539         for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
 540             rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
 541                              pteg, slot);
 542             if (rc != H_SUCCESS) {
 543                 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
 544                 return rc;
 545             }
 546         }
 547         ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
 548     }
 549
 550     return H_SUCCESS;
 551 }
 552
 553 target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
 554                                         SpaprMachineState *spapr,
 555                                         target_ulong flags,
 556                                         target_ulong shift)
 557 {
 558     SpaprPendingHpt *pending = spapr->pending_hpt;
 559     int rc;
 560     size_t newsize;
 561
 562     if (flags != 0) {
 563         return H_PARAMETER;
 564     }
 565
 566     if (!pending || (pending->shift != shift)) {
 567         /* no matching prepare */
 568         return H_CLOSED;
 569     }
 570
 571     if (!pending->complete) {
 572         /* prepare has not completed */
 573         return H_BUSY;
 574     }
 575
 576     /* Shouldn't have got past PREPARE without an HPT */
 577     g_assert(spapr->htab_shift);
 578
 579     newsize = 1ULL << pending->shift;
 580     rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
 581                     pending->hpt, newsize);
 582     if (rc == H_SUCCESS) {
 583         qemu_vfree(spapr->htab);
 584         spapr->htab = pending->hpt;
 585         spapr->htab_shift = pending->shift;
 586
 587         push_sregs_to_kvm_pr(spapr);
 588
 589         pending->hpt = NULL; /* so it's not free()d */
 590     }
 591
 592     /* Clean up */
 593     spapr->pending_hpt = NULL;
 594     free_pending_hpt(pending);
 595
 596     return rc;
 597 }
 598
 599 static void hypercall_register_types(void)
 600 {
 601     /* hcall-pft */
 602     spapr_register_hypercall(H_ENTER, h_enter);
 603     spapr_register_hypercall(H_REMOVE, h_remove);
 604     spapr_register_hypercall(H_PROTECT, h_protect);
 605     spapr_register_hypercall(H_READ, h_read);
 606
 607     /* hcall-bulk */
 608     spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
 609
 610 }
 611
 612 type_init(hypercall_register_types)