2 * QEMU sPAPR PCI for NVLink2 pass through
4 * Copyright (c) 2019 Alexey Kardashevskiy, IBM Corporation.
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 #include "qemu/osdep.h"
25 #include "qapi/error.h"
26 #include "hw/pci/pci.h"
27 #include "hw/pci-host/spapr.h"
28 #include "hw/ppc/spapr_numa.h"
29 #include "qemu/error-report.h"
30 #include "hw/ppc/fdt.h"
31 #include "hw/pci/pci_bridge.h"
33 #define PHANDLE_PCIDEV(phb, pdev) (0x12000000 | \
34 (((phb)->index) << 16) | ((pdev)->devfn))
35 #define PHANDLE_GPURAM(phb, n) (0x110000FF | ((n) << 8) | \
36 (((phb)->index) << 16))
37 #define PHANDLE_NVLINK(phb, gn, nn) (0x00130000 | (((phb)->index) << 8) | \
40 typedef struct SpaprPhbPciNvGpuSlot
{
50 } links
[NVGPU_MAX_LINKS
];
51 } SpaprPhbPciNvGpuSlot
;
53 struct SpaprPhbPciNvGpuConfig
{
54 uint64_t nv2_ram_current
;
55 uint64_t nv2_atsd_current
;
56 int num
; /* number of non empty (i.e. tgt!=0) entries in slots[] */
57 SpaprPhbPciNvGpuSlot slots
[NVGPU_MAX_NUM
];
61 static SpaprPhbPciNvGpuSlot
*
62 spapr_nvgpu_get_slot(SpaprPhbPciNvGpuConfig
*nvgpus
, uint64_t tgt
)
66 /* Search for partially collected "slot" */
67 for (i
= 0; i
< nvgpus
->num
; ++i
) {
68 if (nvgpus
->slots
[i
].tgt
== tgt
) {
69 return &nvgpus
->slots
[i
];
73 if (nvgpus
->num
== ARRAY_SIZE(nvgpus
->slots
)) {
78 nvgpus
->slots
[i
].tgt
= tgt
;
81 return &nvgpus
->slots
[i
];
84 static void spapr_pci_collect_nvgpu(SpaprPhbPciNvGpuConfig
*nvgpus
,
85 PCIDevice
*pdev
, uint64_t tgt
,
86 MemoryRegion
*mr
, Error
**errp
)
88 MachineState
*machine
= MACHINE(qdev_get_machine());
89 SpaprMachineState
*spapr
= SPAPR_MACHINE(machine
);
90 SpaprPhbPciNvGpuSlot
*nvslot
= spapr_nvgpu_get_slot(nvgpus
, tgt
);
93 error_setg(errp
, "Found too many GPUs per vPHB");
96 g_assert(!nvslot
->gpdev
);
99 nvslot
->gpa
= nvgpus
->nv2_ram_current
;
100 nvgpus
->nv2_ram_current
+= memory_region_size(mr
);
101 nvslot
->numa_id
= spapr
->gpu_numa_id
;
102 ++spapr
->gpu_numa_id
;
105 static void spapr_pci_collect_nvnpu(SpaprPhbPciNvGpuConfig
*nvgpus
,
106 PCIDevice
*pdev
, uint64_t tgt
,
107 MemoryRegion
*mr
, Error
**errp
)
109 SpaprPhbPciNvGpuSlot
*nvslot
= spapr_nvgpu_get_slot(nvgpus
, tgt
);
113 error_setg(errp
, "Found too many NVLink bridges per vPHB");
118 if (j
== ARRAY_SIZE(nvslot
->links
)) {
119 error_setg(errp
, "Found too many NVLink bridges per GPU");
124 g_assert(!nvslot
->links
[j
].npdev
);
125 nvslot
->links
[j
].npdev
= pdev
;
126 nvslot
->links
[j
].atsd_gpa
= nvgpus
->nv2_atsd_current
;
127 nvgpus
->nv2_atsd_current
+= memory_region_size(mr
);
128 nvslot
->links
[j
].link_speed
=
129 object_property_get_uint(OBJECT(pdev
), "nvlink2-link-speed", NULL
);
132 static void spapr_phb_pci_collect_nvgpu(PCIBus
*bus
, PCIDevice
*pdev
,
136 Object
*po
= OBJECT(pdev
);
137 uint64_t tgt
= object_property_get_uint(po
, "nvlink2-tgt", NULL
);
140 Error
*local_err
= NULL
;
141 SpaprPhbPciNvGpuConfig
*nvgpus
= opaque
;
142 Object
*mr_gpu
= object_property_get_link(po
, "nvlink2-mr[0]", NULL
);
143 Object
*mr_npu
= object_property_get_link(po
, "nvlink2-atsd-mr[0]",
146 g_assert(mr_gpu
|| mr_npu
);
148 spapr_pci_collect_nvgpu(nvgpus
, pdev
, tgt
, MEMORY_REGION(mr_gpu
),
151 spapr_pci_collect_nvnpu(nvgpus
, pdev
, tgt
, MEMORY_REGION(mr_npu
),
154 error_propagate(&nvgpus
->err
, local_err
);
156 if ((pci_default_read_config(pdev
, PCI_HEADER_TYPE
, 1) !=
157 PCI_HEADER_TYPE_BRIDGE
)) {
161 sec_bus
= pci_bridge_get_sec_bus(PCI_BRIDGE(pdev
));
166 pci_for_each_device_under_bus(sec_bus
, spapr_phb_pci_collect_nvgpu
, opaque
);
169 void spapr_phb_nvgpu_setup(SpaprPhbState
*sphb
, Error
**errp
)
171 int i
, j
, valid_gpu_num
;
174 /* Search for GPUs and NPUs */
175 if (!sphb
->nv2_gpa_win_addr
|| !sphb
->nv2_atsd_win_addr
) {
179 sphb
->nvgpus
= g_new0(SpaprPhbPciNvGpuConfig
, 1);
180 sphb
->nvgpus
->nv2_ram_current
= sphb
->nv2_gpa_win_addr
;
181 sphb
->nvgpus
->nv2_atsd_current
= sphb
->nv2_atsd_win_addr
;
183 bus
= PCI_HOST_BRIDGE(sphb
)->bus
;
184 pci_for_each_device_under_bus(bus
, spapr_phb_pci_collect_nvgpu
,
187 if (sphb
->nvgpus
->err
) {
188 error_propagate(errp
, sphb
->nvgpus
->err
);
189 sphb
->nvgpus
->err
= NULL
;
193 /* Add found GPU RAM and ATSD MRs if found */
194 for (i
= 0, valid_gpu_num
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
196 SpaprPhbPciNvGpuSlot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
198 if (!nvslot
->gpdev
) {
201 nvmrobj
= object_property_get_link(OBJECT(nvslot
->gpdev
),
202 "nvlink2-mr[0]", NULL
);
203 /* ATSD is pointless without GPU RAM MR so skip those */
209 memory_region_add_subregion(get_system_memory(), nvslot
->gpa
,
210 MEMORY_REGION(nvmrobj
));
212 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
215 atsdmrobj
= object_property_get_link(OBJECT(nvslot
->links
[j
].npdev
),
216 "nvlink2-atsd-mr[0]", NULL
);
220 memory_region_add_subregion(get_system_memory(),
221 nvslot
->links
[j
].atsd_gpa
,
222 MEMORY_REGION(atsdmrobj
));
229 /* We did not find any interesting GPU */
231 g_free(sphb
->nvgpus
);
235 void spapr_phb_nvgpu_free(SpaprPhbState
*sphb
)
243 for (i
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
244 SpaprPhbPciNvGpuSlot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
245 Object
*nv_mrobj
= object_property_get_link(OBJECT(nvslot
->gpdev
),
246 "nvlink2-mr[0]", NULL
);
249 memory_region_del_subregion(get_system_memory(),
250 MEMORY_REGION(nv_mrobj
));
252 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
253 PCIDevice
*npdev
= nvslot
->links
[j
].npdev
;
255 atsd_mrobj
= object_property_get_link(OBJECT(npdev
),
256 "nvlink2-atsd-mr[0]", NULL
);
258 memory_region_del_subregion(get_system_memory(),
259 MEMORY_REGION(atsd_mrobj
));
263 g_free(sphb
->nvgpus
);
267 void spapr_phb_nvgpu_populate_dt(SpaprPhbState
*sphb
, void *fdt
, int bus_off
,
270 int i
, j
, atsdnum
= 0;
271 uint64_t atsd
[8]; /* The existing limitation of known guests */
277 for (i
= 0; (i
< sphb
->nvgpus
->num
) && (atsdnum
< ARRAY_SIZE(atsd
)); ++i
) {
278 SpaprPhbPciNvGpuSlot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
280 if (!nvslot
->gpdev
) {
283 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
284 if (!nvslot
->links
[j
].atsd_gpa
) {
288 if (atsdnum
== ARRAY_SIZE(atsd
)) {
289 error_report("Only %"PRIuPTR
" ATSD registers supported",
293 atsd
[atsdnum
] = cpu_to_be64(nvslot
->links
[j
].atsd_gpa
);
299 error_setg(errp
, "No ATSD registers found");
303 if (!spapr_phb_eeh_available(sphb
)) {
305 * ibm,mmio-atsd contains ATSD registers; these belong to an NPU PHB
306 * which we do not emulate as a separate device. Instead we put
307 * ibm,mmio-atsd to the vPHB with GPU and make sure that we do not
308 * put GPUs from different IOMMU groups to the same vPHB to ensure
309 * that the guest will use ATSDs from the corresponding NPU.
311 error_setg(errp
, "ATSD requires separate vPHB per GPU IOMMU group");
315 _FDT((fdt_setprop(fdt
, bus_off
, "ibm,mmio-atsd", atsd
,
316 atsdnum
* sizeof(atsd
[0]))));
319 void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState
*sphb
, void *fdt
)
321 int i
, j
, linkidx
, npuoff
;
322 g_autofree
char *npuname
= NULL
;
328 npuname
= g_strdup_printf("npuphb%d", sphb
->index
);
329 npuoff
= fdt_add_subnode(fdt
, 0, npuname
);
331 _FDT(fdt_setprop_cell(fdt
, npuoff
, "#address-cells", 1));
332 _FDT(fdt_setprop_cell(fdt
, npuoff
, "#size-cells", 0));
333 /* Advertise NPU as POWER9 so the guest can enable NPU2 contexts */
334 _FDT((fdt_setprop_string(fdt
, npuoff
, "compatible", "ibm,power9-npu")));
336 for (i
= 0, linkidx
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
337 for (j
= 0; j
< sphb
->nvgpus
->slots
[i
].linknum
; ++j
) {
338 g_autofree
char *linkname
= g_strdup_printf("link@%d", linkidx
);
339 int off
= fdt_add_subnode(fdt
, npuoff
, linkname
);
342 /* _FDT((fdt_setprop_cell(fdt, off, "reg", linkidx))); */
343 _FDT((fdt_setprop_string(fdt
, off
, "compatible",
345 _FDT((fdt_setprop_cell(fdt
, off
, "phandle",
346 PHANDLE_NVLINK(sphb
, i
, j
))));
347 _FDT((fdt_setprop_cell(fdt
, off
, "ibm,npu-link-index", linkidx
)));
352 /* Add memory nodes for GPU RAM and mark them unusable */
353 for (i
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
354 SpaprPhbPciNvGpuSlot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
355 Object
*nv_mrobj
= object_property_get_link(OBJECT(nvslot
->gpdev
),
358 uint64_t size
= object_property_get_uint(nv_mrobj
, "size", NULL
);
359 uint64_t mem_reg
[2] = { cpu_to_be64(nvslot
->gpa
), cpu_to_be64(size
) };
360 g_autofree
char *mem_name
= g_strdup_printf("memory@%"PRIx64
,
362 int off
= fdt_add_subnode(fdt
, 0, mem_name
);
365 _FDT((fdt_setprop_string(fdt
, off
, "device_type", "memory")));
366 _FDT((fdt_setprop(fdt
, off
, "reg", mem_reg
, sizeof(mem_reg
))));
368 spapr_numa_write_associativity_dt(SPAPR_MACHINE(qdev_get_machine()),
369 fdt
, off
, nvslot
->numa_id
);
371 _FDT((fdt_setprop_string(fdt
, off
, "compatible",
372 "ibm,coherent-device-memory")));
374 mem_reg
[1] = cpu_to_be64(0);
375 _FDT((fdt_setprop(fdt
, off
, "linux,usable-memory", mem_reg
,
377 _FDT((fdt_setprop_cell(fdt
, off
, "phandle",
378 PHANDLE_GPURAM(sphb
, i
))));
383 void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice
*dev
, void *fdt
, int offset
,
392 for (i
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
393 SpaprPhbPciNvGpuSlot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
395 /* Skip "slot" without attached GPU */
396 if (!nvslot
->gpdev
) {
399 if (dev
== nvslot
->gpdev
) {
400 g_autofree
uint32_t *npus
= g_new(uint32_t, nvslot
->linknum
);
402 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
403 PCIDevice
*npdev
= nvslot
->links
[j
].npdev
;
405 npus
[j
] = cpu_to_be32(PHANDLE_PCIDEV(sphb
, npdev
));
407 _FDT(fdt_setprop(fdt
, offset
, "ibm,npu", npus
,
408 j
* sizeof(npus
[0])));
409 _FDT((fdt_setprop_cell(fdt
, offset
, "phandle",
410 PHANDLE_PCIDEV(sphb
, dev
))));
414 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
415 if (dev
!= nvslot
->links
[j
].npdev
) {
419 _FDT((fdt_setprop_cell(fdt
, offset
, "phandle",
420 PHANDLE_PCIDEV(sphb
, dev
))));
421 _FDT(fdt_setprop_cell(fdt
, offset
, "ibm,gpu",
422 PHANDLE_PCIDEV(sphb
, nvslot
->gpdev
)));
423 _FDT((fdt_setprop_cell(fdt
, offset
, "ibm,nvlink",
424 PHANDLE_NVLINK(sphb
, i
, j
))));
426 * If we ever want to emulate GPU RAM at the same location as on
427 * the host - here is the encoding GPA->TGT:
429 * gta = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
430 * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
431 * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
432 * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
434 _FDT(fdt_setprop_cell(fdt
, offset
, "memory-region",
435 PHANDLE_GPURAM(sphb
, i
)));
436 _FDT(fdt_setprop_u64(fdt
, offset
, "ibm,device-tgt-addr",
438 _FDT(fdt_setprop_cell(fdt
, offset
, "ibm,nvlink-speed",
439 nvslot
->links
[j
].link_speed
));