hw/pci-bridge/pci_expander_bridge.c

   1 /*
   2  * PCI Expander Bridge Device Emulation
   3  *
   4  * Copyright (C) 2015 Red Hat Inc
   5  *
   6  * Authors:
   7  *   Marcel Apfelbaum <marcel@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  10  * See the COPYING file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "qapi/error.h"
  15 #include "hw/pci/pci.h"
  16 #include "hw/pci/pci_bus.h"
  17 #include "hw/pci/pci_host.h"
  18 #include "hw/pci/pcie_port.h"
  19 #include "hw/qdev-properties.h"
  20 #include "hw/pci/pci_bridge.h"
  21 #include "hw/pci-bridge/pci_expander_bridge.h"
  22 #include "hw/cxl/cxl.h"
  23 #include "qemu/range.h"
  24 #include "qemu/error-report.h"
  25 #include "qemu/module.h"
  26 #include "sysemu/numa.h"
  27 #include "hw/boards.h"
  28 #include "qom/object.h"
  29
  30 enum BusType { PCI, PCIE, CXL };
  31
  32 #define TYPE_PXB_BUS "pxb-bus"
  33 typedef struct PXBBus PXBBus;
  34 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_BUS,
  35                          TYPE_PXB_BUS)
  36
  37 #define TYPE_PXB_PCIE_BUS "pxb-pcie-bus"
  38 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_PCIE_BUS,
  39                          TYPE_PXB_PCIE_BUS)
  40
  41 #define TYPE_PXB_CXL_BUS "pxb-cxl-bus"
  42 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_CXL_BUS,
  43                          TYPE_PXB_CXL_BUS)
  44
  45 struct PXBBus {
  46     /*< private >*/
  47     PCIBus parent_obj;
  48     /*< public >*/
  49
  50     char bus_path[8];
  51 };
  52
  53 #define TYPE_PXB_PCIE_DEV "pxb-pcie"
  54 OBJECT_DECLARE_SIMPLE_TYPE(PXBPCIEDev, PXB_PCIE_DEV)
  55
  56 static GList *pxb_dev_list;
  57
  58 #define TYPE_PXB_HOST "pxb-host"
  59
  60 CXLComponentState *cxl_get_hb_cstate(PCIHostState *hb)
  61 {
  62     CXLHost *host = PXB_CXL_HOST(hb);
  63
  64     return &host->cxl_cstate;
  65 }
  66
  67 bool cxl_get_hb_passthrough(PCIHostState *hb)
  68 {
  69     CXLHost *host = PXB_CXL_HOST(hb);
  70
  71     return host->passthrough;
  72 }
  73
  74 static int pxb_bus_num(PCIBus *bus)
  75 {
  76     PXBDev *pxb = PXB_DEV(bus->parent_dev);
  77
  78     return pxb->bus_nr;
  79 }
  80
  81 static uint16_t pxb_bus_numa_node(PCIBus *bus)
  82 {
  83     PXBDev *pxb = PXB_DEV(bus->parent_dev);
  84
  85     return pxb->numa_node;
  86 }
  87
  88 static void pxb_bus_class_init(ObjectClass *class, void *data)
  89 {
  90     PCIBusClass *pbc = PCI_BUS_CLASS(class);
  91
  92     pbc->bus_num = pxb_bus_num;
  93     pbc->numa_node = pxb_bus_numa_node;
  94 }
  95
  96 static const TypeInfo pxb_bus_info = {
  97     .name          = TYPE_PXB_BUS,
  98     .parent        = TYPE_PCI_BUS,
  99     .instance_size = sizeof(PXBBus),
 100     .class_init    = pxb_bus_class_init,
 101 };
 102
 103 static const TypeInfo pxb_pcie_bus_info = {
 104     .name          = TYPE_PXB_PCIE_BUS,
 105     .parent        = TYPE_PCIE_BUS,
 106     .instance_size = sizeof(PXBBus),
 107     .class_init    = pxb_bus_class_init,
 108 };
 109
 110 static const TypeInfo pxb_cxl_bus_info = {
 111     .name          = TYPE_PXB_CXL_BUS,
 112     .parent        = TYPE_CXL_BUS,
 113     .instance_size = sizeof(PXBBus),
 114     .class_init    = pxb_bus_class_init,
 115 };
 116
 117 static const char *pxb_host_root_bus_path(PCIHostState *host_bridge,
 118                                           PCIBus *rootbus)
 119 {
 120     PXBBus *bus = pci_bus_is_cxl(rootbus) ?
 121                       PXB_CXL_BUS(rootbus) :
 122                       pci_bus_is_express(rootbus) ? PXB_PCIE_BUS(rootbus) :
 123                                                     PXB_BUS(rootbus);
 124
 125     snprintf(bus->bus_path, 8, "0000:%02x", pxb_bus_num(rootbus));
 126     return bus->bus_path;
 127 }
 128
 129 static char *pxb_host_ofw_unit_address(const SysBusDevice *dev)
 130 {
 131     const PCIHostState *pxb_host;
 132     const PCIBus *pxb_bus;
 133     const PXBDev *pxb_dev;
 134     int position;
 135     const DeviceState *pxb_dev_base;
 136     const PCIHostState *main_host;
 137     const SysBusDevice *main_host_sbd;
 138
 139     pxb_host = PCI_HOST_BRIDGE(dev);
 140     pxb_bus = pxb_host->bus;
 141     pxb_dev = PXB_DEV(pxb_bus->parent_dev);
 142     position = g_list_index(pxb_dev_list, pxb_dev);
 143     assert(position >= 0);
 144
 145     pxb_dev_base = DEVICE(pxb_dev);
 146     main_host = PCI_HOST_BRIDGE(pxb_dev_base->parent_bus->parent);
 147     main_host_sbd = SYS_BUS_DEVICE(main_host);
 148
 149     if (main_host_sbd->num_mmio > 0) {
 150         return g_strdup_printf(HWADDR_FMT_plx ",%x",
 151                                main_host_sbd->mmio[0].addr, position + 1);
 152     }
 153     if (main_host_sbd->num_pio > 0) {
 154         return g_strdup_printf("i%04x,%x",
 155                                main_host_sbd->pio[0], position + 1);
 156     }
 157     return NULL;
 158 }
 159
 160 static void pxb_host_class_init(ObjectClass *class, void *data)
 161 {
 162     DeviceClass *dc = DEVICE_CLASS(class);
 163     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(class);
 164     PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(class);
 165
 166     dc->fw_name = "pci";
 167     /* Reason: Internal part of the pxb/pxb-pcie device, not usable by itself */
 168     dc->user_creatable = false;
 169     sbc->explicit_ofw_unit_address = pxb_host_ofw_unit_address;
 170     hc->root_bus_path = pxb_host_root_bus_path;
 171 }
 172
 173 static const TypeInfo pxb_host_info = {
 174     .name          = TYPE_PXB_HOST,
 175     .parent        = TYPE_PCI_HOST_BRIDGE,
 176     .class_init    = pxb_host_class_init,
 177 };
 178
 179 static void pxb_cxl_realize(DeviceState *dev, Error **errp)
 180 {
 181     SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
 182     CXLHost *cxl = PXB_CXL_HOST(dev);
 183     CXLComponentState *cxl_cstate = &cxl->cxl_cstate;
 184     struct MemoryRegion *mr = &cxl_cstate->crb.component_registers;
 185
 186     cxl_component_register_block_init(OBJECT(dev), cxl_cstate,
 187                                       TYPE_PXB_CXL_HOST);
 188     sysbus_init_mmio(sbd, mr);
 189 }
 190
 191 /*
 192  * Host bridge realization has no means of knowning state associated
 193  * with a particular machine. As such, it is nececssary to delay
 194  * final setup of the host bridge register space until later in the
 195  * machine bring up.
 196  */
 197 void pxb_cxl_hook_up_registers(CXLState *cxl_state, PCIBus *bus, Error **errp)
 198 {
 199     PXBCXLDev *pxb =  PXB_CXL_DEV(pci_bridge_get_device(bus));
 200     CXLHost *cxl = pxb->cxl_host_bridge;
 201     CXLComponentState *cxl_cstate = &cxl->cxl_cstate;
 202     struct MemoryRegion *mr = &cxl_cstate->crb.component_registers;
 203     hwaddr offset;
 204
 205     offset = memory_region_size(mr) * cxl_state->next_mr_idx;
 206     if (offset > memory_region_size(&cxl_state->host_mr)) {
 207         error_setg(errp, "Insufficient space for pxb cxl host register space");
 208         return;
 209     }
 210
 211     memory_region_add_subregion(&cxl_state->host_mr, offset, mr);
 212     cxl_state->next_mr_idx++;
 213 }
 214
 215 static void pxb_cxl_host_class_init(ObjectClass *class, void *data)
 216 {
 217     DeviceClass *dc = DEVICE_CLASS(class);
 218     PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(class);
 219
 220     hc->root_bus_path = pxb_host_root_bus_path;
 221     dc->fw_name = "cxl";
 222     dc->realize = pxb_cxl_realize;
 223     /* Reason: Internal part of the pxb/pxb-pcie device, not usable by itself */
 224     dc->user_creatable = false;
 225 }
 226
 227 /*
 228  * This is a device to handle the MMIO for a CXL host bridge. It does nothing
 229  * else.
 230  */
 231 static const TypeInfo cxl_host_info = {
 232     .name          = TYPE_PXB_CXL_HOST,
 233     .parent        = TYPE_PCI_HOST_BRIDGE,
 234     .instance_size = sizeof(CXLHost),
 235     .class_init    = pxb_cxl_host_class_init,
 236 };
 237
 238 /*
 239  * Registers the PXB bus as a child of pci host root bus.
 240  */
 241 static void pxb_register_bus(PCIDevice *dev, PCIBus *pxb_bus, Error **errp)
 242 {
 243     PCIBus *bus = pci_get_bus(dev);
 244     int pxb_bus_num = pci_bus_num(pxb_bus);
 245
 246     if (bus->parent_dev) {
 247         error_setg(errp, "PXB devices can be attached only to root bus");
 248         return;
 249     }
 250
 251     QLIST_FOREACH(bus, &bus->child, sibling) {
 252         if (pci_bus_num(bus) == pxb_bus_num) {
 253             error_setg(errp, "Bus %d is already in use", pxb_bus_num);
 254             return;
 255         }
 256     }
 257     QLIST_INSERT_HEAD(&pci_get_bus(dev)->child, pxb_bus, sibling);
 258 }
 259
 260 static int pxb_map_irq_fn(PCIDevice *pci_dev, int pin)
 261 {
 262     PCIDevice *pxb = pci_get_bus(pci_dev)->parent_dev;
 263
 264     /*
 265      * First carry out normal swizzle to handle
 266      * multiple root ports on a pxb instance.
 267      */
 268     pin = pci_swizzle_map_irq_fn(pci_dev, pin);
 269
 270     /*
 271      * The bios does not index the pxb slot number when
 272      * it computes the IRQ because it resides on bus 0
 273      * and not on the current bus.
 274      * However QEMU routes the irq through bus 0 and adds
 275      * the pxb slot to the IRQ computation of the PXB
 276      * device.
 277      *
 278      * Synchronize between bios and QEMU by canceling
 279      * pxb's effect.
 280      */
 281     return pin - PCI_SLOT(pxb->devfn);
 282 }
 283
 284 static void pxb_cxl_dev_reset(DeviceState *dev)
 285 {
 286     CXLHost *cxl = PXB_CXL_DEV(dev)->cxl_host_bridge;
 287     CXLComponentState *cxl_cstate = &cxl->cxl_cstate;
 288     PCIHostState *hb = PCI_HOST_BRIDGE(cxl);
 289     uint32_t *reg_state = cxl_cstate->crb.cache_mem_registers;
 290     uint32_t *write_msk = cxl_cstate->crb.cache_mem_regs_write_mask;
 291     int dsp_count = 0;
 292
 293     cxl_component_register_init_common(reg_state, write_msk, CXL2_RC);
 294     /*
 295      * The CXL specification allows for host bridges with no HDM decoders
 296      * if they only have a single root port.
 297      */
 298     if (!PXB_CXL_DEV(dev)->hdm_for_passthrough) {
 299         dsp_count = pcie_count_ds_ports(hb->bus);
 300     }
 301     /* Initial reset will have 0 dsp so wait until > 0 */
 302     if (dsp_count == 1) {
 303         cxl->passthrough = true;
 304         /* Set Capability ID in header to NONE */
 305         ARRAY_FIELD_DP32(reg_state, CXL_HDM_CAPABILITY_HEADER, ID, 0);
 306     } else {
 307         ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, TARGET_COUNT,
 308                          8);
 309     }
 310 }
 311
 312 static gint pxb_compare(gconstpointer a, gconstpointer b)
 313 {
 314     const PXBDev *pxb_a = a, *pxb_b = b;
 315
 316     return pxb_a->bus_nr < pxb_b->bus_nr ? -1 :
 317            pxb_a->bus_nr > pxb_b->bus_nr ?  1 :
 318            0;
 319 }
 320
 321 static void pxb_dev_realize_common(PCIDevice *dev, enum BusType type,
 322                                    Error **errp)
 323 {
 324     PXBDev *pxb = PXB_DEV(dev);
 325     DeviceState *ds, *bds = NULL;
 326     PCIBus *bus;
 327     const char *dev_name = NULL;
 328     Error *local_err = NULL;
 329     MachineState *ms = MACHINE(qdev_get_machine());
 330
 331     if (ms->numa_state == NULL) {
 332         error_setg(errp, "NUMA is not supported by this machine-type");
 333         return;
 334     }
 335
 336     if (pxb->numa_node != NUMA_NODE_UNASSIGNED &&
 337         pxb->numa_node >= ms->numa_state->num_nodes) {
 338         error_setg(errp, "Illegal numa node %d", pxb->numa_node);
 339         return;
 340     }
 341
 342     if (dev->qdev.id && *dev->qdev.id) {
 343         dev_name = dev->qdev.id;
 344     }
 345
 346     ds = qdev_new(type == CXL ? TYPE_PXB_CXL_HOST : TYPE_PXB_HOST);
 347     if (type == PCIE) {
 348         bus = pci_root_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_PCIE_BUS);
 349     } else if (type == CXL) {
 350         bus = pci_root_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_CXL_BUS);
 351         bus->flags |= PCI_BUS_CXL;
 352         PXB_CXL_DEV(dev)->cxl_host_bridge = PXB_CXL_HOST(ds);
 353     } else {
 354         bus = pci_root_bus_new(ds, "pxb-internal", NULL, NULL, 0, TYPE_PXB_BUS);
 355         bds = qdev_new("pci-bridge");
 356         bds->id = g_strdup(dev_name);
 357         qdev_prop_set_uint8(bds, PCI_BRIDGE_DEV_PROP_CHASSIS_NR, pxb->bus_nr);
 358         qdev_prop_set_bit(bds, PCI_BRIDGE_DEV_PROP_SHPC, false);
 359     }
 360
 361     bus->parent_dev = dev;
 362     bus->address_space_mem = pci_get_bus(dev)->address_space_mem;
 363     bus->address_space_io = pci_get_bus(dev)->address_space_io;
 364     bus->map_irq = pxb_map_irq_fn;
 365
 366     PCI_HOST_BRIDGE(ds)->bus = bus;
 367     PCI_HOST_BRIDGE(ds)->bypass_iommu = pxb->bypass_iommu;
 368
 369     pxb_register_bus(dev, bus, &local_err);
 370     if (local_err) {
 371         error_propagate(errp, local_err);
 372         goto err_register_bus;
 373     }
 374
 375     sysbus_realize_and_unref(SYS_BUS_DEVICE(ds), &error_fatal);
 376     if (bds) {
 377         qdev_realize_and_unref(bds, &bus->qbus, &error_fatal);
 378     }
 379
 380     pci_word_test_and_set_mask(dev->config + PCI_STATUS,
 381                                PCI_STATUS_66MHZ | PCI_STATUS_FAST_BACK);
 382     pci_config_set_class(dev->config, PCI_CLASS_BRIDGE_HOST);
 383
 384     pxb_dev_list = g_list_insert_sorted(pxb_dev_list, pxb, pxb_compare);
 385     return;
 386
 387 err_register_bus:
 388     object_unref(OBJECT(bds));
 389     object_unparent(OBJECT(bus));
 390     object_unref(OBJECT(ds));
 391 }
 392
 393 static void pxb_dev_realize(PCIDevice *dev, Error **errp)
 394 {
 395     if (pci_bus_is_express(pci_get_bus(dev))) {
 396         error_setg(errp, "pxb devices cannot reside on a PCIe bus");
 397         return;
 398     }
 399
 400     pxb_dev_realize_common(dev, PCI, errp);
 401 }
 402
 403 static void pxb_dev_exitfn(PCIDevice *pci_dev)
 404 {
 405     PXBDev *pxb = PXB_DEV(pci_dev);
 406
 407     pxb_dev_list = g_list_remove(pxb_dev_list, pxb);
 408 }
 409
 410 static Property pxb_dev_properties[] = {
 411     /* Note: 0 is not a legal PXB bus number. */
 412     DEFINE_PROP_UINT8("bus_nr", PXBDev, bus_nr, 0),
 413     DEFINE_PROP_UINT16("numa_node", PXBDev, numa_node, NUMA_NODE_UNASSIGNED),
 414     DEFINE_PROP_BOOL("bypass_iommu", PXBDev, bypass_iommu, false),
 415     DEFINE_PROP_END_OF_LIST(),
 416 };
 417
 418 static void pxb_dev_class_init(ObjectClass *klass, void *data)
 419 {
 420     DeviceClass *dc = DEVICE_CLASS(klass);
 421     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 422
 423     k->realize = pxb_dev_realize;
 424     k->exit = pxb_dev_exitfn;
 425     k->vendor_id = PCI_VENDOR_ID_REDHAT;
 426     k->device_id = PCI_DEVICE_ID_REDHAT_PXB;
 427     k->class_id = PCI_CLASS_BRIDGE_HOST;
 428
 429     dc->desc = "PCI Expander Bridge";
 430     device_class_set_props(dc, pxb_dev_properties);
 431     dc->hotpluggable = false;
 432     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 433 }
 434
 435 static const TypeInfo pxb_dev_info = {
 436     .name          = TYPE_PXB_DEV,
 437     .parent        = TYPE_PCI_DEVICE,
 438     .instance_size = sizeof(PXBDev),
 439     .class_init    = pxb_dev_class_init,
 440     .interfaces = (InterfaceInfo[]) {
 441         { INTERFACE_CONVENTIONAL_PCI_DEVICE },
 442         { },
 443     },
 444 };
 445
 446 static void pxb_pcie_dev_realize(PCIDevice *dev, Error **errp)
 447 {
 448     if (!pci_bus_is_express(pci_get_bus(dev))) {
 449         error_setg(errp, "pxb-pcie devices cannot reside on a PCI bus");
 450         return;
 451     }
 452
 453     pxb_dev_realize_common(dev, PCIE, errp);
 454 }
 455
 456 static void pxb_pcie_dev_class_init(ObjectClass *klass, void *data)
 457 {
 458     DeviceClass *dc = DEVICE_CLASS(klass);
 459     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 460
 461     k->realize = pxb_pcie_dev_realize;
 462     k->exit = pxb_dev_exitfn;
 463     k->vendor_id = PCI_VENDOR_ID_REDHAT;
 464     k->device_id = PCI_DEVICE_ID_REDHAT_PXB_PCIE;
 465     k->class_id = PCI_CLASS_BRIDGE_HOST;
 466
 467     dc->desc = "PCI Express Expander Bridge";
 468     dc->hotpluggable = false;
 469     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 470 }
 471
 472 static const TypeInfo pxb_pcie_dev_info = {
 473     .name          = TYPE_PXB_PCIE_DEV,
 474     .parent        = TYPE_PXB_DEV,
 475     .instance_size = sizeof(PXBPCIEDev),
 476     .class_init    = pxb_pcie_dev_class_init,
 477     .interfaces = (InterfaceInfo[]) {
 478         { INTERFACE_CONVENTIONAL_PCI_DEVICE },
 479         { },
 480     },
 481 };
 482
 483 static void pxb_cxl_dev_realize(PCIDevice *dev, Error **errp)
 484 {
 485     /* A CXL PXB's parent bus is still PCIe */
 486     if (!pci_bus_is_express(pci_get_bus(dev))) {
 487         error_setg(errp, "pxb-cxl devices cannot reside on a PCI bus");
 488         return;
 489     }
 490
 491     pxb_dev_realize_common(dev, CXL, errp);
 492     pxb_cxl_dev_reset(DEVICE(dev));
 493 }
 494
 495 static Property pxb_cxl_dev_properties[] = {
 496     DEFINE_PROP_BOOL("hdm_for_passthrough", PXBCXLDev, hdm_for_passthrough, false),
 497     DEFINE_PROP_END_OF_LIST(),
 498 };
 499
 500 static void pxb_cxl_dev_class_init(ObjectClass *klass, void *data)
 501 {
 502     DeviceClass *dc   = DEVICE_CLASS(klass);
 503     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 504
 505     k->realize             = pxb_cxl_dev_realize;
 506     k->exit                = pxb_dev_exitfn;
 507     /*
 508      * XXX: These types of bridges don't actually show up in the hierarchy so
 509      * vendor, device, class, etc. ids are intentionally left out.
 510      */
 511
 512     dc->desc = "CXL Host Bridge";
 513     device_class_set_props(dc, pxb_cxl_dev_properties);
 514     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 515
 516     /* Host bridges aren't hotpluggable. FIXME: spec reference */
 517     dc->hotpluggable = false;
 518     dc->reset = pxb_cxl_dev_reset;
 519 }
 520
 521 static const TypeInfo pxb_cxl_dev_info = {
 522     .name          = TYPE_PXB_CXL_DEV,
 523     .parent        = TYPE_PXB_PCIE_DEV,
 524     .instance_size = sizeof(PXBCXLDev),
 525     .class_init    = pxb_cxl_dev_class_init,
 526     .interfaces =
 527         (InterfaceInfo[]){
 528             { INTERFACE_CONVENTIONAL_PCI_DEVICE },
 529             {},
 530         },
 531 };
 532
 533 static void pxb_register_types(void)
 534 {
 535     type_register_static(&pxb_bus_info);
 536     type_register_static(&pxb_pcie_bus_info);
 537     type_register_static(&pxb_cxl_bus_info);
 538     type_register_static(&pxb_host_info);
 539     type_register_static(&cxl_host_info);
 540     type_register_static(&pxb_dev_info);
 541     type_register_static(&pxb_pcie_dev_info);
 542     type_register_static(&pxb_cxl_dev_info);
 543 }
 544
 545 type_init(pxb_register_types)