hw/arm/smmu-common.c

   1 /*
   2  * Copyright (C) 2014-2016 Broadcom Corporation
   3  * Copyright (c) 2017 Red Hat, Inc.
   4  * Written by Prem Mallappa, Eric Auger
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * Author: Prem Mallappa <pmallapp@broadcom.com>
  16  *
  17  */
  18
  19 #include "qemu/osdep.h"
  20 #include "exec/address-spaces.h"
  21 #include "trace.h"
  22 #include "exec/target_page.h"
  23 #include "hw/core/cpu.h"
  24 #include "hw/qdev-properties.h"
  25 #include "qapi/error.h"
  26 #include "qemu/jhash.h"
  27 #include "qemu/module.h"
  28
  29 #include "qemu/error-report.h"
  30 #include "hw/arm/smmu-common.h"
  31 #include "smmu-internal.h"
  32
  33 /* IOTLB Management */
  34
  35 static guint smmu_iotlb_key_hash(gconstpointer v)
  36 {
  37     SMMUIOTLBKey *key = (SMMUIOTLBKey *)v;
  38     uint32_t a, b, c;
  39
  40     /* Jenkins hash */
  41     a = b = c = JHASH_INITVAL + sizeof(*key);
  42     a += key->asid + key->level + key->tg;
  43     b += extract64(key->iova, 0, 32);
  44     c += extract64(key->iova, 32, 32);
  45
  46     __jhash_mix(a, b, c);
  47     __jhash_final(a, b, c);
  48
  49     return c;
  50 }
  51
  52 static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2)
  53 {
  54     SMMUIOTLBKey *k1 = (SMMUIOTLBKey *)v1, *k2 = (SMMUIOTLBKey *)v2;
  55
  56     return (k1->asid == k2->asid) && (k1->iova == k2->iova) &&
  57            (k1->level == k2->level) && (k1->tg == k2->tg);
  58 }
  59
  60 SMMUIOTLBKey smmu_get_iotlb_key(uint16_t asid, uint64_t iova,
  61                                 uint8_t tg, uint8_t level)
  62 {
  63     SMMUIOTLBKey key = {.asid = asid, .iova = iova, .tg = tg, .level = level};
  64
  65     return key;
  66 }
  67
  68 SMMUTLBEntry *smmu_iotlb_lookup(SMMUState *bs, SMMUTransCfg *cfg,
  69                                 SMMUTransTableInfo *tt, hwaddr iova)
  70 {
  71     uint8_t tg = (tt->granule_sz - 10) / 2;
  72     uint8_t inputsize = 64 - tt->tsz;
  73     uint8_t stride = tt->granule_sz - 3;
  74     uint8_t level = 4 - (inputsize - 4) / stride;
  75     SMMUTLBEntry *entry = NULL;
  76
  77     while (level <= 3) {
  78         uint64_t subpage_size = 1ULL << level_shift(level, tt->granule_sz);
  79         uint64_t mask = subpage_size - 1;
  80         SMMUIOTLBKey key;
  81
  82         key = smmu_get_iotlb_key(cfg->asid, iova & ~mask, tg, level);
  83         entry = g_hash_table_lookup(bs->iotlb, &key);
  84         if (entry) {
  85             break;
  86         }
  87         level++;
  88     }
  89
  90     if (entry) {
  91         cfg->iotlb_hits++;
  92         trace_smmu_iotlb_lookup_hit(cfg->asid, iova,
  93                                     cfg->iotlb_hits, cfg->iotlb_misses,
  94                                     100 * cfg->iotlb_hits /
  95                                     (cfg->iotlb_hits + cfg->iotlb_misses));
  96     } else {
  97         cfg->iotlb_misses++;
  98         trace_smmu_iotlb_lookup_miss(cfg->asid, iova,
  99                                      cfg->iotlb_hits, cfg->iotlb_misses,
 100                                      100 * cfg->iotlb_hits /
 101                                      (cfg->iotlb_hits + cfg->iotlb_misses));
 102     }
 103     return entry;
 104 }
 105
 106 void smmu_iotlb_insert(SMMUState *bs, SMMUTransCfg *cfg, SMMUTLBEntry *new)
 107 {
 108     SMMUIOTLBKey *key = g_new0(SMMUIOTLBKey, 1);
 109     uint8_t tg = (new->granule - 10) / 2;
 110
 111     if (g_hash_table_size(bs->iotlb) >= SMMU_IOTLB_MAX_SIZE) {
 112         smmu_iotlb_inv_all(bs);
 113     }
 114
 115     *key = smmu_get_iotlb_key(cfg->asid, new->entry.iova, tg, new->level);
 116     trace_smmu_iotlb_insert(cfg->asid, new->entry.iova, tg, new->level);
 117     g_hash_table_insert(bs->iotlb, key, new);
 118 }
 119
 120 inline void smmu_iotlb_inv_all(SMMUState *s)
 121 {
 122     trace_smmu_iotlb_inv_all();
 123     g_hash_table_remove_all(s->iotlb);
 124 }
 125
 126 static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value,
 127                                          gpointer user_data)
 128 {
 129     uint16_t asid = *(uint16_t *)user_data;
 130     SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key;
 131
 132     return SMMU_IOTLB_ASID(*iotlb_key) == asid;
 133 }
 134
 135 static gboolean smmu_hash_remove_by_asid_iova(gpointer key, gpointer value,
 136                                               gpointer user_data)
 137 {
 138     SMMUTLBEntry *iter = (SMMUTLBEntry *)value;
 139     IOMMUTLBEntry *entry = &iter->entry;
 140     SMMUIOTLBPageInvInfo *info = (SMMUIOTLBPageInvInfo *)user_data;
 141     SMMUIOTLBKey iotlb_key = *(SMMUIOTLBKey *)key;
 142
 143     if (info->asid >= 0 && info->asid != SMMU_IOTLB_ASID(iotlb_key)) {
 144         return false;
 145     }
 146     return ((info->iova & ~entry->addr_mask) == entry->iova) ||
 147            ((entry->iova & ~info->mask) == info->iova);
 148 }
 149
 150 inline void
 151 smmu_iotlb_inv_iova(SMMUState *s, int asid, dma_addr_t iova,
 152                     uint8_t tg, uint64_t num_pages, uint8_t ttl)
 153 {
 154     /* if tg is not set we use 4KB range invalidation */
 155     uint8_t granule = tg ? tg * 2 + 10 : 12;
 156
 157     if (ttl && (num_pages == 1) && (asid >= 0)) {
 158         SMMUIOTLBKey key = smmu_get_iotlb_key(asid, iova, tg, ttl);
 159
 160         if (g_hash_table_remove(s->iotlb, &key)) {
 161             return;
 162         }
 163         /*
 164          * if the entry is not found, let's see if it does not
 165          * belong to a larger IOTLB entry
 166          */
 167     }
 168
 169     SMMUIOTLBPageInvInfo info = {
 170         .asid = asid, .iova = iova,
 171         .mask = (num_pages * 1 << granule) - 1};
 172
 173     g_hash_table_foreach_remove(s->iotlb,
 174                                 smmu_hash_remove_by_asid_iova,
 175                                 &info);
 176 }
 177
 178 inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid)
 179 {
 180     trace_smmu_iotlb_inv_asid(asid);
 181     g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid);
 182 }
 183
 184 /* VMSAv8-64 Translation */
 185
 186 /**
 187  * get_pte - Get the content of a page table entry located at
 188  * @base_addr[@index]
 189  */
 190 static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte,
 191                    SMMUPTWEventInfo *info)
 192 {
 193     int ret;
 194     dma_addr_t addr = baseaddr + index * sizeof(*pte);
 195
 196     /* TODO: guarantee 64-bit single-copy atomicity */
 197     ret = dma_memory_read(&address_space_memory, addr, pte, sizeof(*pte));
 198
 199     if (ret != MEMTX_OK) {
 200         info->type = SMMU_PTW_ERR_WALK_EABT;
 201         info->addr = addr;
 202         return -EINVAL;
 203     }
 204     trace_smmu_get_pte(baseaddr, index, addr, *pte);
 205     return 0;
 206 }
 207
 208 /* VMSAv8-64 Translation Table Format Descriptor Decoding */
 209
 210 /**
 211  * get_page_pte_address - returns the L3 descriptor output address,
 212  * ie. the page frame
 213  * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format
 214  */
 215 static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz)
 216 {
 217     return PTE_ADDRESS(pte, granule_sz);
 218 }
 219
 220 /**
 221  * get_table_pte_address - return table descriptor output address,
 222  * ie. address of next level table
 223  * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats
 224  */
 225 static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz)
 226 {
 227     return PTE_ADDRESS(pte, granule_sz);
 228 }
 229
 230 /**
 231  * get_block_pte_address - return block descriptor output address and block size
 232  * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats
 233  */
 234 static inline hwaddr get_block_pte_address(uint64_t pte, int level,
 235                                            int granule_sz, uint64_t *bsz)
 236 {
 237     int n = level_shift(level, granule_sz);
 238
 239     *bsz = 1ULL << n;
 240     return PTE_ADDRESS(pte, n);
 241 }
 242
 243 SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova)
 244 {
 245     bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi);
 246     uint8_t tbi_byte = tbi * 8;
 247
 248     if (cfg->tt[0].tsz &&
 249         !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) {
 250         /* there is a ttbr0 region and we are in it (high bits all zero) */
 251         return &cfg->tt[0];
 252     } else if (cfg->tt[1].tsz &&
 253            !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) {
 254         /* there is a ttbr1 region and we are in it (high bits all one) */
 255         return &cfg->tt[1];
 256     } else if (!cfg->tt[0].tsz) {
 257         /* ttbr0 region is "everything not in the ttbr1 region" */
 258         return &cfg->tt[0];
 259     } else if (!cfg->tt[1].tsz) {
 260         /* ttbr1 region is "everything not in the ttbr0 region" */
 261         return &cfg->tt[1];
 262     }
 263     /* in the gap between the two regions, this is a Translation fault */
 264     return NULL;
 265 }
 266
 267 /**
 268  * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA
 269  * @cfg: translation config
 270  * @iova: iova to translate
 271  * @perm: access type
 272  * @tlbe: SMMUTLBEntry (out)
 273  * @info: handle to an error info
 274  *
 275  * Return 0 on success, < 0 on error. In case of error, @info is filled
 276  * and tlbe->perm is set to IOMMU_NONE.
 277  * Upon success, @tlbe is filled with translated_addr and entry
 278  * permission rights.
 279  */
 280 static int smmu_ptw_64(SMMUTransCfg *cfg,
 281                        dma_addr_t iova, IOMMUAccessFlags perm,
 282                        SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info)
 283 {
 284     dma_addr_t baseaddr, indexmask;
 285     int stage = cfg->stage;
 286     SMMUTransTableInfo *tt = select_tt(cfg, iova);
 287     uint8_t level, granule_sz, inputsize, stride;
 288
 289     if (!tt || tt->disabled) {
 290         info->type = SMMU_PTW_ERR_TRANSLATION;
 291         goto error;
 292     }
 293
 294     granule_sz = tt->granule_sz;
 295     stride = granule_sz - 3;
 296     inputsize = 64 - tt->tsz;
 297     level = 4 - (inputsize - 4) / stride;
 298     indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1;
 299     baseaddr = extract64(tt->ttb, 0, 48);
 300     baseaddr &= ~indexmask;
 301
 302     while (level <= 3) {
 303         uint64_t subpage_size = 1ULL << level_shift(level, granule_sz);
 304         uint64_t mask = subpage_size - 1;
 305         uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz);
 306         uint64_t pte, gpa;
 307         dma_addr_t pte_addr = baseaddr + offset * sizeof(pte);
 308         uint8_t ap;
 309
 310         if (get_pte(baseaddr, offset, &pte, info)) {
 311                 goto error;
 312         }
 313         trace_smmu_ptw_level(level, iova, subpage_size,
 314                              baseaddr, offset, pte);
 315
 316         if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) {
 317             trace_smmu_ptw_invalid_pte(stage, level, baseaddr,
 318                                        pte_addr, offset, pte);
 319             break;
 320         }
 321
 322         if (is_table_pte(pte, level)) {
 323             ap = PTE_APTABLE(pte);
 324
 325             if (is_permission_fault(ap, perm) && !tt->had) {
 326                 info->type = SMMU_PTW_ERR_PERMISSION;
 327                 goto error;
 328             }
 329             baseaddr = get_table_pte_address(pte, granule_sz);
 330             level++;
 331             continue;
 332         } else if (is_page_pte(pte, level)) {
 333             gpa = get_page_pte_address(pte, granule_sz);
 334             trace_smmu_ptw_page_pte(stage, level, iova,
 335                                     baseaddr, pte_addr, pte, gpa);
 336         } else {
 337             uint64_t block_size;
 338
 339             gpa = get_block_pte_address(pte, level, granule_sz,
 340                                         &block_size);
 341             trace_smmu_ptw_block_pte(stage, level, baseaddr,
 342                                      pte_addr, pte, iova, gpa,
 343                                      block_size >> 20);
 344         }
 345         ap = PTE_AP(pte);
 346         if (is_permission_fault(ap, perm)) {
 347             info->type = SMMU_PTW_ERR_PERMISSION;
 348             goto error;
 349         }
 350
 351         tlbe->entry.translated_addr = gpa;
 352         tlbe->entry.iova = iova & ~mask;
 353         tlbe->entry.addr_mask = mask;
 354         tlbe->entry.perm = PTE_AP_TO_PERM(ap);
 355         tlbe->level = level;
 356         tlbe->granule = granule_sz;
 357         return 0;
 358     }
 359     info->type = SMMU_PTW_ERR_TRANSLATION;
 360
 361 error:
 362     tlbe->entry.perm = IOMMU_NONE;
 363     return -EINVAL;
 364 }
 365
 366 /**
 367  * smmu_ptw - Walk the page tables for an IOVA, according to @cfg
 368  *
 369  * @cfg: translation configuration
 370  * @iova: iova to translate
 371  * @perm: tentative access type
 372  * @tlbe: returned entry
 373  * @info: ptw event handle
 374  *
 375  * return 0 on success
 376  */
 377 inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm,
 378                     SMMUTLBEntry *tlbe, SMMUPTWEventInfo *info)
 379 {
 380     if (!cfg->aa64) {
 381         /*
 382          * This code path is not entered as we check this while decoding
 383          * the configuration data in the derived SMMU model.
 384          */
 385         g_assert_not_reached();
 386     }
 387
 388     return smmu_ptw_64(cfg, iova, perm, tlbe, info);
 389 }
 390
 391 /**
 392  * The bus number is used for lookup when SID based invalidation occurs.
 393  * In that case we lazily populate the SMMUPciBus array from the bus hash
 394  * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus
 395  * numbers may not be always initialized yet.
 396  */
 397 SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num)
 398 {
 399     SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num];
 400     GHashTableIter iter;
 401
 402     if (smmu_pci_bus) {
 403         return smmu_pci_bus;
 404     }
 405
 406     g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr);
 407     while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) {
 408         if (pci_bus_num(smmu_pci_bus->bus) == bus_num) {
 409             s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus;
 410             return smmu_pci_bus;
 411         }
 412     }
 413
 414     return NULL;
 415 }
 416
 417 static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn)
 418 {
 419     SMMUState *s = opaque;
 420     SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus);
 421     SMMUDevice *sdev;
 422     static unsigned int index;
 423
 424     if (!sbus) {
 425         sbus = g_malloc0(sizeof(SMMUPciBus) +
 426                          sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX);
 427         sbus->bus = bus;
 428         g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus);
 429     }
 430
 431     sdev = sbus->pbdev[devfn];
 432     if (!sdev) {
 433         char *name = g_strdup_printf("%s-%d-%d", s->mrtypename, devfn, index++);
 434
 435         sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1);
 436
 437         sdev->smmu = s;
 438         sdev->bus = bus;
 439         sdev->devfn = devfn;
 440
 441         memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu),
 442                                  s->mrtypename,
 443                                  OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS);
 444         address_space_init(&sdev->as,
 445                            MEMORY_REGION(&sdev->iommu), name);
 446         trace_smmu_add_mr(name);
 447         g_free(name);
 448     }
 449
 450     return &sdev->as;
 451 }
 452
 453 IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid)
 454 {
 455     uint8_t bus_n, devfn;
 456     SMMUPciBus *smmu_bus;
 457     SMMUDevice *smmu;
 458
 459     bus_n = PCI_BUS_NUM(sid);
 460     smmu_bus = smmu_find_smmu_pcibus(s, bus_n);
 461     if (smmu_bus) {
 462         devfn = SMMU_PCI_DEVFN(sid);
 463         smmu = smmu_bus->pbdev[devfn];
 464         if (smmu) {
 465             return &smmu->iommu;
 466         }
 467     }
 468     return NULL;
 469 }
 470
 471 /* Unmap the whole notifier's range */
 472 static void smmu_unmap_notifier_range(IOMMUNotifier *n)
 473 {
 474     IOMMUTLBEvent event;
 475
 476     event.type = IOMMU_NOTIFIER_UNMAP;
 477     event.entry.target_as = &address_space_memory;
 478     event.entry.iova = n->start;
 479     event.entry.perm = IOMMU_NONE;
 480     event.entry.addr_mask = n->end - n->start;
 481
 482     memory_region_notify_iommu_one(n, &event);
 483 }
 484
 485 /* Unmap all notifiers attached to @mr */
 486 inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr)
 487 {
 488     IOMMUNotifier *n;
 489
 490     trace_smmu_inv_notifiers_mr(mr->parent_obj.name);
 491     IOMMU_NOTIFIER_FOREACH(n, mr) {
 492         smmu_unmap_notifier_range(n);
 493     }
 494 }
 495
 496 /* Unmap all notifiers of all mr's */
 497 void smmu_inv_notifiers_all(SMMUState *s)
 498 {
 499     SMMUDevice *sdev;
 500
 501     QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) {
 502         smmu_inv_notifiers_mr(&sdev->iommu);
 503     }
 504 }
 505
 506 static void smmu_base_realize(DeviceState *dev, Error **errp)
 507 {
 508     SMMUState *s = ARM_SMMU(dev);
 509     SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev);
 510     Error *local_err = NULL;
 511
 512     sbc->parent_realize(dev, &local_err);
 513     if (local_err) {
 514         error_propagate(errp, local_err);
 515         return;
 516     }
 517     s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free);
 518     s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal,
 519                                      g_free, g_free);
 520     s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL);
 521
 522     if (s->primary_bus) {
 523         pci_setup_iommu(s->primary_bus, smmu_find_add_as, s);
 524     } else {
 525         error_setg(errp, "SMMU is not attached to any PCI bus!");
 526     }
 527 }
 528
 529 static void smmu_base_reset(DeviceState *dev)
 530 {
 531     SMMUState *s = ARM_SMMU(dev);
 532
 533     g_hash_table_remove_all(s->configs);
 534     g_hash_table_remove_all(s->iotlb);
 535 }
 536
 537 static Property smmu_dev_properties[] = {
 538     DEFINE_PROP_UINT8("bus_num", SMMUState, bus_num, 0),
 539     DEFINE_PROP_LINK("primary-bus", SMMUState, primary_bus, "PCI", PCIBus *),
 540     DEFINE_PROP_END_OF_LIST(),
 541 };
 542
 543 static void smmu_base_class_init(ObjectClass *klass, void *data)
 544 {
 545     DeviceClass *dc = DEVICE_CLASS(klass);
 546     SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass);
 547
 548     device_class_set_props(dc, smmu_dev_properties);
 549     device_class_set_parent_realize(dc, smmu_base_realize,
 550                                     &sbc->parent_realize);
 551     dc->reset = smmu_base_reset;
 552 }
 553
 554 static const TypeInfo smmu_base_info = {
 555     .name          = TYPE_ARM_SMMU,
 556     .parent        = TYPE_SYS_BUS_DEVICE,
 557     .instance_size = sizeof(SMMUState),
 558     .class_data    = NULL,
 559     .class_size    = sizeof(SMMUBaseClass),
 560     .class_init    = smmu_base_class_init,
 561     .abstract      = true,
 562 };
 563
 564 static void smmu_base_register_types(void)
 565 {
 566     type_register_static(&smmu_base_info);
 567 }
 568
 569 type_init(smmu_base_register_types)
 570