hw/ppc/spapr_softmmu.c

   1 #include "qemu/osdep.h"
   2 #include "qemu/cutils.h"
   3 #include "qemu/memalign.h"
   4 #include "cpu.h"
   5 #include "helper_regs.h"
   6 #include "hw/ppc/spapr.h"
   7 #include "mmu-hash64.h"
   8 #include "mmu-book3s-v3.h"
   9
  10 static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
  11 {
  12     /*
  13      * hash value/pteg group index is normalized by HPT mask
  14      */
  15     if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
  16         return false;
  17     }
  18     return true;
  19 }
  20
  21 static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
  22                             target_ulong opcode, target_ulong *args)
  23 {
  24     target_ulong flags = args[0];
  25     target_ulong ptex = args[1];
  26     target_ulong pteh = args[2];
  27     target_ulong ptel = args[3];
  28     unsigned apshift;
  29     target_ulong raddr;
  30     target_ulong slot;
  31     const ppc_hash_pte64_t *hptes;
  32
  33     apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
  34     if (!apshift) {
  35         /* Bad page size encoding */
  36         return H_PARAMETER;
  37     }
  38
  39     raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
  40
  41     if (is_ram_address(spapr, raddr)) {
  42         /* Regular RAM - should have WIMG=0010 */
  43         if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
  44             return H_PARAMETER;
  45         }
  46     } else {
  47         target_ulong wimg_flags;
  48         /* Looks like an IO address */
  49         /* FIXME: What WIMG combinations could be sensible for IO?
  50          * For now we allow WIMG=010x, but are there others? */
  51         /* FIXME: Should we check against registered IO addresses? */
  52         wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
  53
  54         if (wimg_flags != HPTE64_R_I &&
  55             wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
  56             return H_PARAMETER;
  57         }
  58     }
  59
  60     pteh &= ~0x60ULL;
  61
  62     if (!valid_ptex(cpu, ptex)) {
  63         return H_PARAMETER;
  64     }
  65
  66     slot = ptex & 7ULL;
  67     ptex = ptex & ~7ULL;
  68
  69     if (likely((flags & H_EXACT) == 0)) {
  70         hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
  71         for (slot = 0; slot < 8; slot++) {
  72             if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
  73                 break;
  74             }
  75         }
  76         ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
  77         if (slot == 8) {
  78             return H_PTEG_FULL;
  79         }
  80     } else {
  81         hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
  82         if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
  83             ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
  84             return H_PTEG_FULL;
  85         }
  86         ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
  87     }
  88
  89     spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
  90
  91     args[0] = ptex + slot;
  92     return H_SUCCESS;
  93 }
  94
  95 typedef enum {
  96     REMOVE_SUCCESS = 0,
  97     REMOVE_NOT_FOUND = 1,
  98     REMOVE_PARM = 2,
  99     REMOVE_HW = 3,
 100 } RemoveResult;
 101
 102 static RemoveResult remove_hpte(PowerPCCPU *cpu
 103                                 , target_ulong ptex,
 104                                 target_ulong avpn,
 105                                 target_ulong flags,
 106                                 target_ulong *vp, target_ulong *rp)
 107 {
 108     const ppc_hash_pte64_t *hptes;
 109     target_ulong v, r;
 110
 111     if (!valid_ptex(cpu, ptex)) {
 112         return REMOVE_PARM;
 113     }
 114
 115     hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
 116     v = ppc_hash64_hpte0(cpu, hptes, 0);
 117     r = ppc_hash64_hpte1(cpu, hptes, 0);
 118     ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
 119
 120     if ((v & HPTE64_V_VALID) == 0 ||
 121         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
 122         ((flags & H_ANDCOND) && (v & avpn) != 0)) {
 123         return REMOVE_NOT_FOUND;
 124     }
 125     *vp = v;
 126     *rp = r;
 127     spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
 128     ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
 129     return REMOVE_SUCCESS;
 130 }
 131
 132 static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
 133                              target_ulong opcode, target_ulong *args)
 134 {
 135     CPUPPCState *env = &cpu->env;
 136     target_ulong flags = args[0];
 137     target_ulong ptex = args[1];
 138     target_ulong avpn = args[2];
 139     RemoveResult ret;
 140
 141     ret = remove_hpte(cpu, ptex, avpn, flags,
 142                       &args[0], &args[1]);
 143
 144     switch (ret) {
 145     case REMOVE_SUCCESS:
 146         check_tlb_flush(env, true);
 147         return H_SUCCESS;
 148
 149     case REMOVE_NOT_FOUND:
 150         return H_NOT_FOUND;
 151
 152     case REMOVE_PARM:
 153         return H_PARAMETER;
 154
 155     case REMOVE_HW:
 156         return H_HARDWARE;
 157     }
 158
 159     g_assert_not_reached();
 160 }
 161
 162 #define H_BULK_REMOVE_TYPE             0xc000000000000000ULL
 163 #define   H_BULK_REMOVE_REQUEST        0x4000000000000000ULL
 164 #define   H_BULK_REMOVE_RESPONSE       0x8000000000000000ULL
 165 #define   H_BULK_REMOVE_END            0xc000000000000000ULL
 166 #define H_BULK_REMOVE_CODE             0x3000000000000000ULL
 167 #define   H_BULK_REMOVE_SUCCESS        0x0000000000000000ULL
 168 #define   H_BULK_REMOVE_NOT_FOUND      0x1000000000000000ULL
 169 #define   H_BULK_REMOVE_PARM           0x2000000000000000ULL
 170 #define   H_BULK_REMOVE_HW             0x3000000000000000ULL
 171 #define H_BULK_REMOVE_RC               0x0c00000000000000ULL
 172 #define H_BULK_REMOVE_FLAGS            0x0300000000000000ULL
 173 #define   H_BULK_REMOVE_ABSOLUTE       0x0000000000000000ULL
 174 #define   H_BULK_REMOVE_ANDCOND        0x0100000000000000ULL
 175 #define   H_BULK_REMOVE_AVPN           0x0200000000000000ULL
 176 #define H_BULK_REMOVE_PTEX             0x00ffffffffffffffULL
 177
 178 #define H_BULK_REMOVE_MAX_BATCH        4
 179
 180 static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
 181                                   target_ulong opcode, target_ulong *args)
 182 {
 183     CPUPPCState *env = &cpu->env;
 184     int i;
 185     target_ulong rc = H_SUCCESS;
 186
 187     for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
 188         target_ulong *tsh = &args[i*2];
 189         target_ulong tsl = args[i*2 + 1];
 190         target_ulong v, r, ret;
 191
 192         if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
 193             break;
 194         } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
 195             return H_PARAMETER;
 196         }
 197
 198         *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
 199         *tsh |= H_BULK_REMOVE_RESPONSE;
 200
 201         if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
 202             *tsh |= H_BULK_REMOVE_PARM;
 203             return H_PARAMETER;
 204         }
 205
 206         ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
 207                           (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
 208                           &v, &r);
 209
 210         *tsh |= ret << 60;
 211
 212         switch (ret) {
 213         case REMOVE_SUCCESS:
 214             *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
 215             break;
 216
 217         case REMOVE_PARM:
 218             rc = H_PARAMETER;
 219             goto exit;
 220
 221         case REMOVE_HW:
 222             rc = H_HARDWARE;
 223             goto exit;
 224         }
 225     }
 226  exit:
 227     check_tlb_flush(env, true);
 228
 229     return rc;
 230 }
 231
 232 static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
 233                               target_ulong opcode, target_ulong *args)
 234 {
 235     CPUPPCState *env = &cpu->env;
 236     target_ulong flags = args[0];
 237     target_ulong ptex = args[1];
 238     target_ulong avpn = args[2];
 239     const ppc_hash_pte64_t *hptes;
 240     target_ulong v, r;
 241
 242     if (!valid_ptex(cpu, ptex)) {
 243         return H_PARAMETER;
 244     }
 245
 246     hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
 247     v = ppc_hash64_hpte0(cpu, hptes, 0);
 248     r = ppc_hash64_hpte1(cpu, hptes, 0);
 249     ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
 250
 251     if ((v & HPTE64_V_VALID) == 0 ||
 252         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
 253         return H_NOT_FOUND;
 254     }
 255
 256     r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
 257            HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
 258     r |= (flags << 55) & HPTE64_R_PP0;
 259     r |= (flags << 48) & HPTE64_R_KEY_HI;
 260     r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
 261     spapr_store_hpte(cpu, ptex,
 262                      (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
 263     ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
 264     /* Flush the tlb */
 265     check_tlb_flush(env, true);
 266     /* Don't need a memory barrier, due to qemu's global lock */
 267     spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
 268     return H_SUCCESS;
 269 }
 270
 271 static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
 272                            target_ulong opcode, target_ulong *args)
 273 {
 274     target_ulong flags = args[0];
 275     target_ulong ptex = args[1];
 276     int i, ridx, n_entries = 1;
 277     const ppc_hash_pte64_t *hptes;
 278
 279     if (!valid_ptex(cpu, ptex)) {
 280         return H_PARAMETER;
 281     }
 282
 283     if (flags & H_READ_4) {
 284         /* Clear the two low order bits */
 285         ptex &= ~(3ULL);
 286         n_entries = 4;
 287     }
 288
 289     hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
 290     for (i = 0, ridx = 0; i < n_entries; i++) {
 291         args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
 292         args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
 293     }
 294     ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
 295
 296     return H_SUCCESS;
 297 }
 298
 299 struct SpaprPendingHpt {
 300     /* These fields are read-only after initialization */
 301     int shift;
 302     QemuThread thread;
 303
 304     /* These fields are protected by the BQL */
 305     bool complete;
 306
 307     /* These fields are private to the preparation thread if
 308      * !complete, otherwise protected by the BQL */
 309     int ret;
 310     void *hpt;
 311 };
 312
 313 static void free_pending_hpt(SpaprPendingHpt *pending)
 314 {
 315     if (pending->hpt) {
 316         qemu_vfree(pending->hpt);
 317     }
 318
 319     g_free(pending);
 320 }
 321
 322 static void *hpt_prepare_thread(void *opaque)
 323 {
 324     SpaprPendingHpt *pending = opaque;
 325     size_t size = 1ULL << pending->shift;
 326
 327     pending->hpt = qemu_try_memalign(size, size);
 328     if (pending->hpt) {
 329         memset(pending->hpt, 0, size);
 330         pending->ret = H_SUCCESS;
 331     } else {
 332         pending->ret = H_NO_MEM;
 333     }
 334
 335     qemu_mutex_lock_iothread();
 336
 337     if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
 338         /* Ready to go */
 339         pending->complete = true;
 340     } else {
 341         /* We've been cancelled, clean ourselves up */
 342         free_pending_hpt(pending);
 343     }
 344
 345     qemu_mutex_unlock_iothread();
 346     return NULL;
 347 }
 348
 349 /* Must be called with BQL held */
 350 static void cancel_hpt_prepare(SpaprMachineState *spapr)
 351 {
 352     SpaprPendingHpt *pending = spapr->pending_hpt;
 353
 354     /* Let the thread know it's cancelled */
 355     spapr->pending_hpt = NULL;
 356
 357     if (!pending) {
 358         /* Nothing to do */
 359         return;
 360     }
 361
 362     if (!pending->complete) {
 363         /* thread will clean itself up */
 364         return;
 365     }
 366
 367     free_pending_hpt(pending);
 368 }
 369
 370 target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
 371                                          SpaprMachineState *spapr,
 372                                          target_ulong shift)
 373 {
 374     SpaprPendingHpt *pending = spapr->pending_hpt;
 375
 376     if (pending) {
 377         /* something already in progress */
 378         if (pending->shift == shift) {
 379             /* and it's suitable */
 380             if (pending->complete) {
 381                 return pending->ret;
 382             } else {
 383                 return H_LONG_BUSY_ORDER_100_MSEC;
 384             }
 385         }
 386
 387         /* not suitable, cancel and replace */
 388         cancel_hpt_prepare(spapr);
 389     }
 390
 391     if (!shift) {
 392         /* nothing to do */
 393         return H_SUCCESS;
 394     }
 395
 396     /* start new prepare */
 397
 398     pending = g_new0(SpaprPendingHpt, 1);
 399     pending->shift = shift;
 400     pending->ret = H_HARDWARE;
 401
 402     qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
 403                        hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
 404
 405     spapr->pending_hpt = pending;
 406
 407     /* In theory we could estimate the time more accurately based on
 408      * the new size, but there's not much point */
 409     return H_LONG_BUSY_ORDER_100_MSEC;
 410 }
 411
 412 static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
 413 {
 414     uint8_t *addr = htab;
 415
 416     addr += pteg * HASH_PTEG_SIZE_64;
 417     addr += slot * HASH_PTE_SIZE_64;
 418     return  ldq_p(addr);
 419 }
 420
 421 static void new_hpte_store(void *htab, uint64_t pteg, int slot,
 422                            uint64_t pte0, uint64_t pte1)
 423 {
 424     uint8_t *addr = htab;
 425
 426     addr += pteg * HASH_PTEG_SIZE_64;
 427     addr += slot * HASH_PTE_SIZE_64;
 428
 429     stq_p(addr, pte0);
 430     stq_p(addr + HPTE64_DW1, pte1);
 431 }
 432
 433 static int rehash_hpte(PowerPCCPU *cpu,
 434                        const ppc_hash_pte64_t *hptes,
 435                        void *old_hpt, uint64_t oldsize,
 436                        void *new_hpt, uint64_t newsize,
 437                        uint64_t pteg, int slot)
 438 {
 439     uint64_t old_hash_mask = (oldsize >> 7) - 1;
 440     uint64_t new_hash_mask = (newsize >> 7) - 1;
 441     target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
 442     target_ulong pte1;
 443     uint64_t avpn;
 444     unsigned base_pg_shift;
 445     uint64_t hash, new_pteg, replace_pte0;
 446
 447     if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
 448         return H_SUCCESS;
 449     }
 450
 451     pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
 452
 453     base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
 454     assert(base_pg_shift); /* H_ENTER shouldn't allow a bad encoding */
 455     avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
 456
 457     if (pte0 & HPTE64_V_SECONDARY) {
 458         pteg = ~pteg;
 459     }
 460
 461     if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
 462         uint64_t offset, vsid;
 463
 464         /* We only have 28 - 23 bits of offset in avpn */
 465         offset = (avpn & 0x1f) << 23;
 466         vsid = avpn >> 5;
 467         /* We can find more bits from the pteg value */
 468         if (base_pg_shift < 23) {
 469             offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
 470         }
 471
 472         hash = vsid ^ (offset >> base_pg_shift);
 473     } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
 474         uint64_t offset, vsid;
 475
 476         /* We only have 40 - 23 bits of seg_off in avpn */
 477         offset = (avpn & 0x1ffff) << 23;
 478         vsid = avpn >> 17;
 479         if (base_pg_shift < 23) {
 480             offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
 481                 << base_pg_shift;
 482         }
 483
 484         hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
 485     } else {
 486         error_report("rehash_pte: Bad segment size in HPTE");
 487         return H_HARDWARE;
 488     }
 489
 490     new_pteg = hash & new_hash_mask;
 491     if (pte0 & HPTE64_V_SECONDARY) {
 492         assert(~pteg == (hash & old_hash_mask));
 493         new_pteg = ~new_pteg;
 494     } else {
 495         assert(pteg == (hash & old_hash_mask));
 496     }
 497     assert((oldsize != newsize) || (pteg == new_pteg));
 498     replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
 499     /*
 500      * Strictly speaking, we don't need all these tests, since we only
 501      * ever rehash bolted HPTEs.  We might in future handle non-bolted
 502      * HPTEs, though so make the logic correct for those cases as
 503      * well.
 504      */
 505     if (replace_pte0 & HPTE64_V_VALID) {
 506         assert(newsize < oldsize);
 507         if (replace_pte0 & HPTE64_V_BOLTED) {
 508             if (pte0 & HPTE64_V_BOLTED) {
 509                 /* Bolted collision, nothing we can do */
 510                 return H_PTEG_FULL;
 511             } else {
 512                 /* Discard this hpte */
 513                 return H_SUCCESS;
 514             }
 515         }
 516     }
 517
 518     new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
 519     return H_SUCCESS;
 520 }
 521
 522 static int rehash_hpt(PowerPCCPU *cpu,
 523                       void *old_hpt, uint64_t oldsize,
 524                       void *new_hpt, uint64_t newsize)
 525 {
 526     uint64_t n_ptegs = oldsize >> 7;
 527     uint64_t pteg;
 528     int slot;
 529     int rc;
 530
 531     for (pteg = 0; pteg < n_ptegs; pteg++) {
 532         hwaddr ptex = pteg * HPTES_PER_GROUP;
 533         const ppc_hash_pte64_t *hptes
 534             = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
 535
 536         if (!hptes) {
 537             return H_HARDWARE;
 538         }
 539
 540         for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
 541             rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
 542                              pteg, slot);
 543             if (rc != H_SUCCESS) {
 544                 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
 545                 return rc;
 546             }
 547         }
 548         ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
 549     }
 550
 551     return H_SUCCESS;
 552 }
 553
 554 target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
 555                                         SpaprMachineState *spapr,
 556                                         target_ulong flags,
 557                                         target_ulong shift)
 558 {
 559     SpaprPendingHpt *pending = spapr->pending_hpt;
 560     int rc;
 561     size_t newsize;
 562
 563     if (flags != 0) {
 564         return H_PARAMETER;
 565     }
 566
 567     if (!pending || (pending->shift != shift)) {
 568         /* no matching prepare */
 569         return H_CLOSED;
 570     }
 571
 572     if (!pending->complete) {
 573         /* prepare has not completed */
 574         return H_BUSY;
 575     }
 576
 577     /* Shouldn't have got past PREPARE without an HPT */
 578     g_assert(spapr->htab_shift);
 579
 580     newsize = 1ULL << pending->shift;
 581     rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
 582                     pending->hpt, newsize);
 583     if (rc == H_SUCCESS) {
 584         qemu_vfree(spapr->htab);
 585         spapr->htab = pending->hpt;
 586         spapr->htab_shift = pending->shift;
 587
 588         push_sregs_to_kvm_pr(spapr);
 589
 590         pending->hpt = NULL; /* so it's not free()d */
 591     }
 592
 593     /* Clean up */
 594     spapr->pending_hpt = NULL;
 595     free_pending_hpt(pending);
 596
 597     return rc;
 598 }
 599
 600 static void hypercall_register_types(void)
 601 {
 602     /* hcall-pft */
 603     spapr_register_hypercall(H_ENTER, h_enter);
 604     spapr_register_hypercall(H_REMOVE, h_remove);
 605     spapr_register_hypercall(H_PROTECT, h_protect);
 606     spapr_register_hypercall(H_READ, h_read);
 607
 608     /* hcall-bulk */
 609     spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
 610
 611 }
 612
 613 type_init(hypercall_register_types)