2 * QEMU sPAPR PCI for NVLink2 pass through
4 * Copyright (c) 2019 Alexey Kardashevskiy, IBM Corporation.
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24 #include "qemu/osdep.h"
25 #include "qapi/error.h"
26 #include "qemu-common.h"
27 #include "hw/pci/pci.h"
28 #include "hw/pci-host/spapr.h"
29 #include "qemu/error-report.h"
30 #include "hw/ppc/fdt.h"
31 #include "hw/pci/pci_bridge.h"
33 #define PHANDLE_PCIDEV(phb, pdev) (0x12000000 | \
34 (((phb)->index) << 16) | ((pdev)->devfn))
35 #define PHANDLE_GPURAM(phb, n) (0x110000FF | ((n) << 8) | \
36 (((phb)->index) << 16))
37 #define PHANDLE_NVLINK(phb, gn, nn) (0x00130000 | (((phb)->index) << 8) | \
40 #define SPAPR_GPU_NUMA_ID (cpu_to_be32(1))
42 struct spapr_phb_pci_nvgpu_config
{
43 uint64_t nv2_ram_current
;
44 uint64_t nv2_atsd_current
;
45 int num
; /* number of non empty (i.e. tgt!=0) entries in slots[] */
46 struct spapr_phb_pci_nvgpu_slot
{
56 } links
[NVGPU_MAX_LINKS
];
57 } slots
[NVGPU_MAX_NUM
];
61 static struct spapr_phb_pci_nvgpu_slot
*
62 spapr_nvgpu_get_slot(struct spapr_phb_pci_nvgpu_config
*nvgpus
, uint64_t tgt
)
66 /* Search for partially collected "slot" */
67 for (i
= 0; i
< nvgpus
->num
; ++i
) {
68 if (nvgpus
->slots
[i
].tgt
== tgt
) {
69 return &nvgpus
->slots
[i
];
73 if (nvgpus
->num
== ARRAY_SIZE(nvgpus
->slots
)) {
78 nvgpus
->slots
[i
].tgt
= tgt
;
81 return &nvgpus
->slots
[i
];
84 static void spapr_pci_collect_nvgpu(struct spapr_phb_pci_nvgpu_config
*nvgpus
,
85 PCIDevice
*pdev
, uint64_t tgt
,
86 MemoryRegion
*mr
, Error
**errp
)
88 MachineState
*machine
= MACHINE(qdev_get_machine());
89 SpaprMachineState
*spapr
= SPAPR_MACHINE(machine
);
90 struct spapr_phb_pci_nvgpu_slot
*nvslot
= spapr_nvgpu_get_slot(nvgpus
, tgt
);
93 error_setg(errp
, "Found too many GPUs per vPHB");
96 g_assert(!nvslot
->gpdev
);
99 nvslot
->gpa
= nvgpus
->nv2_ram_current
;
100 nvgpus
->nv2_ram_current
+= memory_region_size(mr
);
101 nvslot
->numa_id
= spapr
->gpu_numa_id
;
102 ++spapr
->gpu_numa_id
;
105 static void spapr_pci_collect_nvnpu(struct spapr_phb_pci_nvgpu_config
*nvgpus
,
106 PCIDevice
*pdev
, uint64_t tgt
,
107 MemoryRegion
*mr
, Error
**errp
)
109 struct spapr_phb_pci_nvgpu_slot
*nvslot
= spapr_nvgpu_get_slot(nvgpus
, tgt
);
113 error_setg(errp
, "Found too many NVLink bridges per vPHB");
118 if (j
== ARRAY_SIZE(nvslot
->links
)) {
119 error_setg(errp
, "Found too many NVLink bridges per GPU");
124 g_assert(!nvslot
->links
[j
].npdev
);
125 nvslot
->links
[j
].npdev
= pdev
;
126 nvslot
->links
[j
].atsd_gpa
= nvgpus
->nv2_atsd_current
;
127 nvgpus
->nv2_atsd_current
+= memory_region_size(mr
);
128 nvslot
->links
[j
].link_speed
=
129 object_property_get_uint(OBJECT(pdev
), "nvlink2-link-speed", NULL
);
132 static void spapr_phb_pci_collect_nvgpu(PCIBus
*bus
, PCIDevice
*pdev
,
136 Object
*po
= OBJECT(pdev
);
137 uint64_t tgt
= object_property_get_uint(po
, "nvlink2-tgt", NULL
);
140 Error
*local_err
= NULL
;
141 struct spapr_phb_pci_nvgpu_config
*nvgpus
= opaque
;
142 Object
*mr_gpu
= object_property_get_link(po
, "nvlink2-mr[0]", NULL
);
143 Object
*mr_npu
= object_property_get_link(po
, "nvlink2-atsd-mr[0]",
146 g_assert(mr_gpu
|| mr_npu
);
148 spapr_pci_collect_nvgpu(nvgpus
, pdev
, tgt
, MEMORY_REGION(mr_gpu
),
151 spapr_pci_collect_nvnpu(nvgpus
, pdev
, tgt
, MEMORY_REGION(mr_npu
),
154 error_propagate(&nvgpus
->errp
, local_err
);
156 if ((pci_default_read_config(pdev
, PCI_HEADER_TYPE
, 1) !=
157 PCI_HEADER_TYPE_BRIDGE
)) {
161 sec_bus
= pci_bridge_get_sec_bus(PCI_BRIDGE(pdev
));
166 pci_for_each_device(sec_bus
, pci_bus_num(sec_bus
),
167 spapr_phb_pci_collect_nvgpu
, opaque
);
170 void spapr_phb_nvgpu_setup(SpaprPhbState
*sphb
, Error
**errp
)
172 int i
, j
, valid_gpu_num
;
175 /* Search for GPUs and NPUs */
176 if (!sphb
->nv2_gpa_win_addr
|| !sphb
->nv2_atsd_win_addr
) {
180 sphb
->nvgpus
= g_new0(struct spapr_phb_pci_nvgpu_config
, 1);
181 sphb
->nvgpus
->nv2_ram_current
= sphb
->nv2_gpa_win_addr
;
182 sphb
->nvgpus
->nv2_atsd_current
= sphb
->nv2_atsd_win_addr
;
184 bus
= PCI_HOST_BRIDGE(sphb
)->bus
;
185 pci_for_each_device(bus
, pci_bus_num(bus
),
186 spapr_phb_pci_collect_nvgpu
, sphb
->nvgpus
);
188 if (sphb
->nvgpus
->errp
) {
189 error_propagate(errp
, sphb
->nvgpus
->errp
);
190 sphb
->nvgpus
->errp
= NULL
;
194 /* Add found GPU RAM and ATSD MRs if found */
195 for (i
= 0, valid_gpu_num
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
197 struct spapr_phb_pci_nvgpu_slot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
199 if (!nvslot
->gpdev
) {
202 nvmrobj
= object_property_get_link(OBJECT(nvslot
->gpdev
),
203 "nvlink2-mr[0]", NULL
);
204 /* ATSD is pointless without GPU RAM MR so skip those */
210 memory_region_add_subregion(get_system_memory(), nvslot
->gpa
,
211 MEMORY_REGION(nvmrobj
));
213 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
216 atsdmrobj
= object_property_get_link(OBJECT(nvslot
->links
[j
].npdev
),
217 "nvlink2-atsd-mr[0]", NULL
);
221 memory_region_add_subregion(get_system_memory(),
222 nvslot
->links
[j
].atsd_gpa
,
223 MEMORY_REGION(atsdmrobj
));
230 /* We did not find any interesting GPU */
232 g_free(sphb
->nvgpus
);
236 void spapr_phb_nvgpu_free(SpaprPhbState
*sphb
)
244 for (i
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
245 struct spapr_phb_pci_nvgpu_slot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
246 Object
*nv_mrobj
= object_property_get_link(OBJECT(nvslot
->gpdev
),
247 "nvlink2-mr[0]", NULL
);
250 memory_region_del_subregion(get_system_memory(),
251 MEMORY_REGION(nv_mrobj
));
253 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
254 PCIDevice
*npdev
= nvslot
->links
[j
].npdev
;
256 atsd_mrobj
= object_property_get_link(OBJECT(npdev
),
257 "nvlink2-atsd-mr[0]", NULL
);
259 memory_region_del_subregion(get_system_memory(),
260 MEMORY_REGION(atsd_mrobj
));
264 g_free(sphb
->nvgpus
);
268 void spapr_phb_nvgpu_populate_dt(SpaprPhbState
*sphb
, void *fdt
, int bus_off
,
271 int i
, j
, atsdnum
= 0;
272 uint64_t atsd
[8]; /* The existing limitation of known guests */
278 for (i
= 0; (i
< sphb
->nvgpus
->num
) && (atsdnum
< ARRAY_SIZE(atsd
)); ++i
) {
279 struct spapr_phb_pci_nvgpu_slot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
281 if (!nvslot
->gpdev
) {
284 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
285 if (!nvslot
->links
[j
].atsd_gpa
) {
289 if (atsdnum
== ARRAY_SIZE(atsd
)) {
290 error_report("Only %"PRIuPTR
" ATSD registers supported",
294 atsd
[atsdnum
] = cpu_to_be64(nvslot
->links
[j
].atsd_gpa
);
300 error_setg(errp
, "No ATSD registers found");
304 if (!spapr_phb_eeh_available(sphb
)) {
306 * ibm,mmio-atsd contains ATSD registers; these belong to an NPU PHB
307 * which we do not emulate as a separate device. Instead we put
308 * ibm,mmio-atsd to the vPHB with GPU and make sure that we do not
309 * put GPUs from different IOMMU groups to the same vPHB to ensure
310 * that the guest will use ATSDs from the corresponding NPU.
312 error_setg(errp
, "ATSD requires separate vPHB per GPU IOMMU group");
316 _FDT((fdt_setprop(fdt
, bus_off
, "ibm,mmio-atsd", atsd
,
317 atsdnum
* sizeof(atsd
[0]))));
320 void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState
*sphb
, void *fdt
)
322 int i
, j
, linkidx
, npuoff
;
329 npuname
= g_strdup_printf("npuphb%d", sphb
->index
);
330 npuoff
= fdt_add_subnode(fdt
, 0, npuname
);
332 _FDT(fdt_setprop_cell(fdt
, npuoff
, "#address-cells", 1));
333 _FDT(fdt_setprop_cell(fdt
, npuoff
, "#size-cells", 0));
334 /* Advertise NPU as POWER9 so the guest can enable NPU2 contexts */
335 _FDT((fdt_setprop_string(fdt
, npuoff
, "compatible", "ibm,power9-npu")));
338 for (i
= 0, linkidx
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
339 for (j
= 0; j
< sphb
->nvgpus
->slots
[i
].linknum
; ++j
) {
340 char *linkname
= g_strdup_printf("link@%d", linkidx
);
341 int off
= fdt_add_subnode(fdt
, npuoff
, linkname
);
344 /* _FDT((fdt_setprop_cell(fdt, off, "reg", linkidx))); */
345 _FDT((fdt_setprop_string(fdt
, off
, "compatible",
347 _FDT((fdt_setprop_cell(fdt
, off
, "phandle",
348 PHANDLE_NVLINK(sphb
, i
, j
))));
349 _FDT((fdt_setprop_cell(fdt
, off
, "ibm,npu-link-index", linkidx
)));
355 /* Add memory nodes for GPU RAM and mark them unusable */
356 for (i
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
357 struct spapr_phb_pci_nvgpu_slot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
358 Object
*nv_mrobj
= object_property_get_link(OBJECT(nvslot
->gpdev
),
359 "nvlink2-mr[0]", NULL
);
360 uint32_t associativity
[] = {
365 cpu_to_be32(nvslot
->numa_id
)
367 uint64_t size
= object_property_get_uint(nv_mrobj
, "size", NULL
);
368 uint64_t mem_reg
[2] = { cpu_to_be64(nvslot
->gpa
), cpu_to_be64(size
) };
369 char *mem_name
= g_strdup_printf("memory@%"PRIx64
, nvslot
->gpa
);
370 int off
= fdt_add_subnode(fdt
, 0, mem_name
);
373 _FDT((fdt_setprop_string(fdt
, off
, "device_type", "memory")));
374 _FDT((fdt_setprop(fdt
, off
, "reg", mem_reg
, sizeof(mem_reg
))));
375 _FDT((fdt_setprop(fdt
, off
, "ibm,associativity", associativity
,
376 sizeof(associativity
))));
378 _FDT((fdt_setprop_string(fdt
, off
, "compatible",
379 "ibm,coherent-device-memory")));
381 mem_reg
[1] = cpu_to_be64(0);
382 _FDT((fdt_setprop(fdt
, off
, "linux,usable-memory", mem_reg
,
384 _FDT((fdt_setprop_cell(fdt
, off
, "phandle",
385 PHANDLE_GPURAM(sphb
, i
))));
391 void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice
*dev
, void *fdt
, int offset
,
400 for (i
= 0; i
< sphb
->nvgpus
->num
; ++i
) {
401 struct spapr_phb_pci_nvgpu_slot
*nvslot
= &sphb
->nvgpus
->slots
[i
];
403 /* Skip "slot" without attached GPU */
404 if (!nvslot
->gpdev
) {
407 if (dev
== nvslot
->gpdev
) {
408 uint32_t npus
[nvslot
->linknum
];
410 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
411 PCIDevice
*npdev
= nvslot
->links
[j
].npdev
;
413 npus
[j
] = cpu_to_be32(PHANDLE_PCIDEV(sphb
, npdev
));
415 _FDT(fdt_setprop(fdt
, offset
, "ibm,npu", npus
,
416 j
* sizeof(npus
[0])));
417 _FDT((fdt_setprop_cell(fdt
, offset
, "phandle",
418 PHANDLE_PCIDEV(sphb
, dev
))));
422 for (j
= 0; j
< nvslot
->linknum
; ++j
) {
423 if (dev
!= nvslot
->links
[j
].npdev
) {
427 _FDT((fdt_setprop_cell(fdt
, offset
, "phandle",
428 PHANDLE_PCIDEV(sphb
, dev
))));
429 _FDT(fdt_setprop_cell(fdt
, offset
, "ibm,gpu",
430 PHANDLE_PCIDEV(sphb
, nvslot
->gpdev
)));
431 _FDT((fdt_setprop_cell(fdt
, offset
, "ibm,nvlink",
432 PHANDLE_NVLINK(sphb
, i
, j
))));
434 * If we ever want to emulate GPU RAM at the same location as on
435 * the host - here is the encoding GPA->TGT:
437 * gta = ((sphb->nv2_gpa >> 42) & 0x1) << 42;
438 * gta |= ((sphb->nv2_gpa >> 45) & 0x3) << 43;
439 * gta |= ((sphb->nv2_gpa >> 49) & 0x3) << 45;
440 * gta |= sphb->nv2_gpa & ((1UL << 43) - 1);
442 _FDT(fdt_setprop_cell(fdt
, offset
, "memory-region",
443 PHANDLE_GPURAM(sphb
, i
)));
444 _FDT(fdt_setprop_u64(fdt
, offset
, "ibm,device-tgt-addr",
446 _FDT(fdt_setprop_cell(fdt
, offset
, "ibm,nvlink-speed",
447 nvslot
->links
[j
].link_speed
));