hw/pci-bridge/pci_expander_bridge.c

   1 /*
   2  * PCI Expander Bridge Device Emulation
   3  *
   4  * Copyright (C) 2015 Red Hat Inc
   5  *
   6  * Authors:
   7  *   Marcel Apfelbaum <marcel@redhat.com>
   8  *
   9  * This work is licensed under the terms of the GNU GPL, version 2 or later.
  10  * See the COPYING file in the top-level directory.
  11  */
  12
  13 #include "qemu/osdep.h"
  14 #include "qapi/error.h"
  15 #include "hw/pci/pci.h"
  16 #include "hw/pci/pci_bus.h"
  17 #include "hw/pci/pci_host.h"
  18 #include "hw/qdev-properties.h"
  19 #include "hw/pci/pci_bridge.h"
  20 #include "hw/cxl/cxl.h"
  21 #include "qemu/range.h"
  22 #include "qemu/error-report.h"
  23 #include "qemu/module.h"
  24 #include "sysemu/numa.h"
  25 #include "hw/boards.h"
  26 #include "qom/object.h"
  27
  28 enum BusType { PCI, PCIE, CXL };
  29
  30 #define TYPE_PXB_BUS "pxb-bus"
  31 typedef struct PXBBus PXBBus;
  32 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_BUS,
  33                          TYPE_PXB_BUS)
  34
  35 #define TYPE_PXB_PCIE_BUS "pxb-pcie-bus"
  36 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_PCIE_BUS,
  37                          TYPE_PXB_PCIE_BUS)
  38
  39 #define TYPE_PXB_CXL_BUS "pxb-cxl-bus"
  40 DECLARE_INSTANCE_CHECKER(PXBBus, PXB_CXL_BUS,
  41                          TYPE_PXB_CXL_BUS)
  42
  43 struct PXBBus {
  44     /*< private >*/
  45     PCIBus parent_obj;
  46     /*< public >*/
  47
  48     char bus_path[8];
  49 };
  50
  51 #define TYPE_PXB_DEVICE "pxb"
  52 typedef struct PXBDev PXBDev;
  53 DECLARE_INSTANCE_CHECKER(PXBDev, PXB_DEV,
  54                          TYPE_PXB_DEVICE)
  55
  56 #define TYPE_PXB_PCIE_DEVICE "pxb-pcie"
  57 DECLARE_INSTANCE_CHECKER(PXBDev, PXB_PCIE_DEV,
  58                          TYPE_PXB_PCIE_DEVICE)
  59
  60 static PXBDev *convert_to_pxb(PCIDevice *dev)
  61 {
  62     /* A CXL PXB's parent bus is PCIe, so the normal check won't work */
  63     if (object_dynamic_cast(OBJECT(dev), TYPE_PXB_CXL_DEVICE)) {
  64         return PXB_CXL_DEV(dev);
  65     }
  66
  67     return pci_bus_is_express(pci_get_bus(dev))
  68         ? PXB_PCIE_DEV(dev) : PXB_DEV(dev);
  69 }
  70
  71 static GList *pxb_dev_list;
  72
  73 #define TYPE_PXB_HOST "pxb-host"
  74
  75 CXLComponentState *cxl_get_hb_cstate(PCIHostState *hb)
  76 {
  77     CXLHost *host = PXB_CXL_HOST(hb);
  78
  79     return &host->cxl_cstate;
  80 }
  81
  82 static int pxb_bus_num(PCIBus *bus)
  83 {
  84     PXBDev *pxb = convert_to_pxb(bus->parent_dev);
  85
  86     return pxb->bus_nr;
  87 }
  88
  89 static uint16_t pxb_bus_numa_node(PCIBus *bus)
  90 {
  91     PXBDev *pxb = convert_to_pxb(bus->parent_dev);
  92
  93     return pxb->numa_node;
  94 }
  95
  96 static void pxb_bus_class_init(ObjectClass *class, void *data)
  97 {
  98     PCIBusClass *pbc = PCI_BUS_CLASS(class);
  99
 100     pbc->bus_num = pxb_bus_num;
 101     pbc->numa_node = pxb_bus_numa_node;
 102 }
 103
 104 static const TypeInfo pxb_bus_info = {
 105     .name          = TYPE_PXB_BUS,
 106     .parent        = TYPE_PCI_BUS,
 107     .instance_size = sizeof(PXBBus),
 108     .class_init    = pxb_bus_class_init,
 109 };
 110
 111 static const TypeInfo pxb_pcie_bus_info = {
 112     .name          = TYPE_PXB_PCIE_BUS,
 113     .parent        = TYPE_PCIE_BUS,
 114     .instance_size = sizeof(PXBBus),
 115     .class_init    = pxb_bus_class_init,
 116 };
 117
 118 static const TypeInfo pxb_cxl_bus_info = {
 119     .name          = TYPE_PXB_CXL_BUS,
 120     .parent        = TYPE_CXL_BUS,
 121     .instance_size = sizeof(PXBBus),
 122     .class_init    = pxb_bus_class_init,
 123 };
 124
 125 static const char *pxb_host_root_bus_path(PCIHostState *host_bridge,
 126                                           PCIBus *rootbus)
 127 {
 128     PXBBus *bus = pci_bus_is_cxl(rootbus) ?
 129                       PXB_CXL_BUS(rootbus) :
 130                       pci_bus_is_express(rootbus) ? PXB_PCIE_BUS(rootbus) :
 131                                                     PXB_BUS(rootbus);
 132
 133     snprintf(bus->bus_path, 8, "0000:%02x", pxb_bus_num(rootbus));
 134     return bus->bus_path;
 135 }
 136
 137 static char *pxb_host_ofw_unit_address(const SysBusDevice *dev)
 138 {
 139     const PCIHostState *pxb_host;
 140     const PCIBus *pxb_bus;
 141     const PXBDev *pxb_dev;
 142     int position;
 143     const DeviceState *pxb_dev_base;
 144     const PCIHostState *main_host;
 145     const SysBusDevice *main_host_sbd;
 146
 147     pxb_host = PCI_HOST_BRIDGE(dev);
 148     pxb_bus = pxb_host->bus;
 149     pxb_dev = convert_to_pxb(pxb_bus->parent_dev);
 150     position = g_list_index(pxb_dev_list, pxb_dev);
 151     assert(position >= 0);
 152
 153     pxb_dev_base = DEVICE(pxb_dev);
 154     main_host = PCI_HOST_BRIDGE(pxb_dev_base->parent_bus->parent);
 155     main_host_sbd = SYS_BUS_DEVICE(main_host);
 156
 157     if (main_host_sbd->num_mmio > 0) {
 158         return g_strdup_printf(TARGET_FMT_plx ",%x",
 159                                main_host_sbd->mmio[0].addr, position + 1);
 160     }
 161     if (main_host_sbd->num_pio > 0) {
 162         return g_strdup_printf("i%04x,%x",
 163                                main_host_sbd->pio[0], position + 1);
 164     }
 165     return NULL;
 166 }
 167
 168 static void pxb_host_class_init(ObjectClass *class, void *data)
 169 {
 170     DeviceClass *dc = DEVICE_CLASS(class);
 171     SysBusDeviceClass *sbc = SYS_BUS_DEVICE_CLASS(class);
 172     PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(class);
 173
 174     dc->fw_name = "pci";
 175     /* Reason: Internal part of the pxb/pxb-pcie device, not usable by itself */
 176     dc->user_creatable = false;
 177     sbc->explicit_ofw_unit_address = pxb_host_ofw_unit_address;
 178     hc->root_bus_path = pxb_host_root_bus_path;
 179 }
 180
 181 static const TypeInfo pxb_host_info = {
 182     .name          = TYPE_PXB_HOST,
 183     .parent        = TYPE_PCI_HOST_BRIDGE,
 184     .class_init    = pxb_host_class_init,
 185 };
 186
 187 static void pxb_cxl_realize(DeviceState *dev, Error **errp)
 188 {
 189     MachineState *ms = MACHINE(qdev_get_machine());
 190     SysBusDevice *sbd = SYS_BUS_DEVICE(dev);
 191     CXLHost *cxl = PXB_CXL_HOST(dev);
 192     CXLComponentState *cxl_cstate = &cxl->cxl_cstate;
 193     struct MemoryRegion *mr = &cxl_cstate->crb.component_registers;
 194     hwaddr offset;
 195
 196     cxl_component_register_block_init(OBJECT(dev), cxl_cstate,
 197                                       TYPE_PXB_CXL_HOST);
 198     sysbus_init_mmio(sbd, mr);
 199
 200     offset = memory_region_size(mr) * ms->cxl_devices_state->next_mr_idx;
 201     if (offset > memory_region_size(&ms->cxl_devices_state->host_mr)) {
 202         error_setg(errp, "Insufficient space for pxb cxl host register space");
 203         return;
 204     }
 205
 206     memory_region_add_subregion(&ms->cxl_devices_state->host_mr, offset, mr);
 207     ms->cxl_devices_state->next_mr_idx++;
 208 }
 209
 210 static void pxb_cxl_host_class_init(ObjectClass *class, void *data)
 211 {
 212     DeviceClass *dc = DEVICE_CLASS(class);
 213     PCIHostBridgeClass *hc = PCI_HOST_BRIDGE_CLASS(class);
 214
 215     hc->root_bus_path = pxb_host_root_bus_path;
 216     dc->fw_name = "cxl";
 217     dc->realize = pxb_cxl_realize;
 218     /* Reason: Internal part of the pxb/pxb-pcie device, not usable by itself */
 219     dc->user_creatable = false;
 220 }
 221
 222 /*
 223  * This is a device to handle the MMIO for a CXL host bridge. It does nothing
 224  * else.
 225  */
 226 static const TypeInfo cxl_host_info = {
 227     .name          = TYPE_PXB_CXL_HOST,
 228     .parent        = TYPE_PCI_HOST_BRIDGE,
 229     .instance_size = sizeof(CXLHost),
 230     .class_init    = pxb_cxl_host_class_init,
 231 };
 232
 233 /*
 234  * Registers the PXB bus as a child of pci host root bus.
 235  */
 236 static void pxb_register_bus(PCIDevice *dev, PCIBus *pxb_bus, Error **errp)
 237 {
 238     PCIBus *bus = pci_get_bus(dev);
 239     int pxb_bus_num = pci_bus_num(pxb_bus);
 240
 241     if (bus->parent_dev) {
 242         error_setg(errp, "PXB devices can be attached only to root bus");
 243         return;
 244     }
 245
 246     QLIST_FOREACH(bus, &bus->child, sibling) {
 247         if (pci_bus_num(bus) == pxb_bus_num) {
 248             error_setg(errp, "Bus %d is already in use", pxb_bus_num);
 249             return;
 250         }
 251     }
 252     QLIST_INSERT_HEAD(&pci_get_bus(dev)->child, pxb_bus, sibling);
 253 }
 254
 255 static int pxb_map_irq_fn(PCIDevice *pci_dev, int pin)
 256 {
 257     PCIDevice *pxb = pci_get_bus(pci_dev)->parent_dev;
 258
 259     /*
 260      * First carry out normal swizzle to handle
 261      * multple root ports on a pxb instance.
 262      */
 263     pin = pci_swizzle_map_irq_fn(pci_dev, pin);
 264
 265     /*
 266      * The bios does not index the pxb slot number when
 267      * it computes the IRQ because it resides on bus 0
 268      * and not on the current bus.
 269      * However QEMU routes the irq through bus 0 and adds
 270      * the pxb slot to the IRQ computation of the PXB
 271      * device.
 272      *
 273      * Synchronize between bios and QEMU by canceling
 274      * pxb's effect.
 275      */
 276     return pin - PCI_SLOT(pxb->devfn);
 277 }
 278
 279 static void pxb_dev_reset(DeviceState *dev)
 280 {
 281     CXLHost *cxl = PXB_CXL_DEV(dev)->cxl.cxl_host_bridge;
 282     CXLComponentState *cxl_cstate = &cxl->cxl_cstate;
 283     uint32_t *reg_state = cxl_cstate->crb.cache_mem_registers;
 284     uint32_t *write_msk = cxl_cstate->crb.cache_mem_regs_write_mask;
 285
 286     cxl_component_register_init_common(reg_state, write_msk, CXL2_ROOT_PORT);
 287     ARRAY_FIELD_DP32(reg_state, CXL_HDM_DECODER_CAPABILITY, TARGET_COUNT, 8);
 288 }
 289
 290 static gint pxb_compare(gconstpointer a, gconstpointer b)
 291 {
 292     const PXBDev *pxb_a = a, *pxb_b = b;
 293
 294     return pxb_a->bus_nr < pxb_b->bus_nr ? -1 :
 295            pxb_a->bus_nr > pxb_b->bus_nr ?  1 :
 296            0;
 297 }
 298
 299 static void pxb_dev_realize_common(PCIDevice *dev, enum BusType type,
 300                                    Error **errp)
 301 {
 302     PXBDev *pxb = convert_to_pxb(dev);
 303     DeviceState *ds, *bds = NULL;
 304     PCIBus *bus;
 305     const char *dev_name = NULL;
 306     Error *local_err = NULL;
 307     MachineState *ms = MACHINE(qdev_get_machine());
 308
 309     if (ms->numa_state == NULL) {
 310         error_setg(errp, "NUMA is not supported by this machine-type");
 311         return;
 312     }
 313
 314     if (pxb->numa_node != NUMA_NODE_UNASSIGNED &&
 315         pxb->numa_node >= ms->numa_state->num_nodes) {
 316         error_setg(errp, "Illegal numa node %d", pxb->numa_node);
 317         return;
 318     }
 319
 320     if (dev->qdev.id && *dev->qdev.id) {
 321         dev_name = dev->qdev.id;
 322     }
 323
 324     ds = qdev_new(type == CXL ? TYPE_PXB_CXL_HOST : TYPE_PXB_HOST);
 325     if (type == PCIE) {
 326         bus = pci_root_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_PCIE_BUS);
 327     } else if (type == CXL) {
 328         bus = pci_root_bus_new(ds, dev_name, NULL, NULL, 0, TYPE_PXB_CXL_BUS);
 329         bus->flags |= PCI_BUS_CXL;
 330         PXB_CXL_DEV(dev)->cxl.cxl_host_bridge = PXB_CXL_HOST(ds);
 331     } else {
 332         bus = pci_root_bus_new(ds, "pxb-internal", NULL, NULL, 0, TYPE_PXB_BUS);
 333         bds = qdev_new("pci-bridge");
 334         bds->id = g_strdup(dev_name);
 335         qdev_prop_set_uint8(bds, PCI_BRIDGE_DEV_PROP_CHASSIS_NR, pxb->bus_nr);
 336         qdev_prop_set_bit(bds, PCI_BRIDGE_DEV_PROP_SHPC, false);
 337     }
 338
 339     bus->parent_dev = dev;
 340     bus->address_space_mem = pci_get_bus(dev)->address_space_mem;
 341     bus->address_space_io = pci_get_bus(dev)->address_space_io;
 342     bus->map_irq = pxb_map_irq_fn;
 343
 344     PCI_HOST_BRIDGE(ds)->bus = bus;
 345     PCI_HOST_BRIDGE(ds)->bypass_iommu = pxb->bypass_iommu;
 346
 347     pxb_register_bus(dev, bus, &local_err);
 348     if (local_err) {
 349         error_propagate(errp, local_err);
 350         goto err_register_bus;
 351     }
 352
 353     sysbus_realize_and_unref(SYS_BUS_DEVICE(ds), &error_fatal);
 354     if (bds) {
 355         qdev_realize_and_unref(bds, &bus->qbus, &error_fatal);
 356     }
 357
 358     pci_word_test_and_set_mask(dev->config + PCI_STATUS,
 359                                PCI_STATUS_66MHZ | PCI_STATUS_FAST_BACK);
 360     pci_config_set_class(dev->config, PCI_CLASS_BRIDGE_HOST);
 361
 362     pxb_dev_list = g_list_insert_sorted(pxb_dev_list, pxb, pxb_compare);
 363     return;
 364
 365 err_register_bus:
 366     object_unref(OBJECT(bds));
 367     object_unparent(OBJECT(bus));
 368     object_unref(OBJECT(ds));
 369 }
 370
 371 static void pxb_dev_realize(PCIDevice *dev, Error **errp)
 372 {
 373     if (pci_bus_is_express(pci_get_bus(dev))) {
 374         error_setg(errp, "pxb devices cannot reside on a PCIe bus");
 375         return;
 376     }
 377
 378     pxb_dev_realize_common(dev, PCI, errp);
 379 }
 380
 381 static void pxb_dev_exitfn(PCIDevice *pci_dev)
 382 {
 383     PXBDev *pxb = convert_to_pxb(pci_dev);
 384
 385     pxb_dev_list = g_list_remove(pxb_dev_list, pxb);
 386 }
 387
 388 static Property pxb_dev_properties[] = {
 389     /* Note: 0 is not a legal PXB bus number. */
 390     DEFINE_PROP_UINT8("bus_nr", PXBDev, bus_nr, 0),
 391     DEFINE_PROP_UINT16("numa_node", PXBDev, numa_node, NUMA_NODE_UNASSIGNED),
 392     DEFINE_PROP_BOOL("bypass_iommu", PXBDev, bypass_iommu, false),
 393     DEFINE_PROP_END_OF_LIST(),
 394 };
 395
 396 static void pxb_dev_class_init(ObjectClass *klass, void *data)
 397 {
 398     DeviceClass *dc = DEVICE_CLASS(klass);
 399     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 400
 401     k->realize = pxb_dev_realize;
 402     k->exit = pxb_dev_exitfn;
 403     k->vendor_id = PCI_VENDOR_ID_REDHAT;
 404     k->device_id = PCI_DEVICE_ID_REDHAT_PXB;
 405     k->class_id = PCI_CLASS_BRIDGE_HOST;
 406
 407     dc->desc = "PCI Expander Bridge";
 408     device_class_set_props(dc, pxb_dev_properties);
 409     dc->hotpluggable = false;
 410     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 411 }
 412
 413 static const TypeInfo pxb_dev_info = {
 414     .name          = TYPE_PXB_DEVICE,
 415     .parent        = TYPE_PCI_DEVICE,
 416     .instance_size = sizeof(PXBDev),
 417     .class_init    = pxb_dev_class_init,
 418     .interfaces = (InterfaceInfo[]) {
 419         { INTERFACE_CONVENTIONAL_PCI_DEVICE },
 420         { },
 421     },
 422 };
 423
 424 static void pxb_pcie_dev_realize(PCIDevice *dev, Error **errp)
 425 {
 426     if (!pci_bus_is_express(pci_get_bus(dev))) {
 427         error_setg(errp, "pxb-pcie devices cannot reside on a PCI bus");
 428         return;
 429     }
 430
 431     pxb_dev_realize_common(dev, PCIE, errp);
 432 }
 433
 434 static void pxb_pcie_dev_class_init(ObjectClass *klass, void *data)
 435 {
 436     DeviceClass *dc = DEVICE_CLASS(klass);
 437     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 438
 439     k->realize = pxb_pcie_dev_realize;
 440     k->exit = pxb_dev_exitfn;
 441     k->vendor_id = PCI_VENDOR_ID_REDHAT;
 442     k->device_id = PCI_DEVICE_ID_REDHAT_PXB_PCIE;
 443     k->class_id = PCI_CLASS_BRIDGE_HOST;
 444
 445     dc->desc = "PCI Express Expander Bridge";
 446     device_class_set_props(dc, pxb_dev_properties);
 447     dc->hotpluggable = false;
 448     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 449 }
 450
 451 static const TypeInfo pxb_pcie_dev_info = {
 452     .name          = TYPE_PXB_PCIE_DEVICE,
 453     .parent        = TYPE_PCI_DEVICE,
 454     .instance_size = sizeof(PXBDev),
 455     .class_init    = pxb_pcie_dev_class_init,
 456     .interfaces = (InterfaceInfo[]) {
 457         { INTERFACE_CONVENTIONAL_PCI_DEVICE },
 458         { },
 459     },
 460 };
 461
 462 static void pxb_cxl_dev_realize(PCIDevice *dev, Error **errp)
 463 {
 464     MachineState *ms = MACHINE(qdev_get_machine());
 465
 466     /* A CXL PXB's parent bus is still PCIe */
 467     if (!pci_bus_is_express(pci_get_bus(dev))) {
 468         error_setg(errp, "pxb-cxl devices cannot reside on a PCI bus");
 469         return;
 470     }
 471     if (!ms->cxl_devices_state->is_enabled) {
 472         error_setg(errp, "Machine does not have cxl=on");
 473         return;
 474     }
 475
 476     pxb_dev_realize_common(dev, CXL, errp);
 477     pxb_dev_reset(DEVICE(dev));
 478 }
 479
 480 static void pxb_cxl_dev_class_init(ObjectClass *klass, void *data)
 481 {
 482     DeviceClass *dc   = DEVICE_CLASS(klass);
 483     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 484
 485     k->realize             = pxb_cxl_dev_realize;
 486     k->exit                = pxb_dev_exitfn;
 487     /*
 488      * XXX: These types of bridges don't actually show up in the hierarchy so
 489      * vendor, device, class, etc. ids are intentionally left out.
 490      */
 491
 492     dc->desc = "CXL Host Bridge";
 493     device_class_set_props(dc, pxb_dev_properties);
 494     set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories);
 495
 496     /* Host bridges aren't hotpluggable. FIXME: spec reference */
 497     dc->hotpluggable = false;
 498     dc->reset = pxb_dev_reset;
 499 }
 500
 501 static const TypeInfo pxb_cxl_dev_info = {
 502     .name          = TYPE_PXB_CXL_DEVICE,
 503     .parent        = TYPE_PCI_DEVICE,
 504     .instance_size = sizeof(PXBDev),
 505     .class_init    = pxb_cxl_dev_class_init,
 506     .interfaces =
 507         (InterfaceInfo[]){
 508             { INTERFACE_CONVENTIONAL_PCI_DEVICE },
 509             {},
 510         },
 511 };
 512
 513 static void pxb_register_types(void)
 514 {
 515     type_register_static(&pxb_bus_info);
 516     type_register_static(&pxb_pcie_bus_info);
 517     type_register_static(&pxb_cxl_bus_info);
 518     type_register_static(&pxb_host_info);
 519     type_register_static(&cxl_host_info);
 520     type_register_static(&pxb_dev_info);
 521     type_register_static(&pxb_pcie_dev_info);
 522     type_register_static(&pxb_cxl_dev_info);
 523 }
 524
 525 type_init(pxb_register_types)