1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#include "qemu/osdep.h"
25#include "qapi/error.h"
26#include "qemu-common.h"
27#include "hw/pci/pci.h"
28#include "hw/pci-host/spapr.h"
29#include "qemu/error-report.h"
30#include "hw/ppc/fdt.h"
31#include "hw/pci/pci_bridge.h"
32
33#define PHANDLE_PCIDEV(phb, pdev) (0x12000000 | \
34 (((phb)->index) << 16) | ((pdev)->devfn))
35#define PHANDLE_GPURAM(phb, n) (0x110000FF | ((n) << 8) | \
36 (((phb)->index) << 16))
37#define PHANDLE_NVLINK(phb, gn, nn) (0x00130000 | (((phb)->index) << 8) | \
38 ((gn) << 4) | (nn))
39
40#define SPAPR_GPU_NUMA_ID (cpu_to_be32(1))
41
42struct spapr_phb_pci_nvgpu_config {
43 uint64_t nv2_ram_current;
44 uint64_t nv2_atsd_current;
45 int num;
46 struct spapr_phb_pci_nvgpu_slot {
47 uint64_t tgt;
48 uint64_t gpa;
49 unsigned numa_id;
50 PCIDevice *gpdev;
51 int linknum;
52 struct {
53 uint64_t atsd_gpa;
54 PCIDevice *npdev;
55 uint32_t link_speed;
56 } links[NVGPU_MAX_LINKS];
57 } slots[NVGPU_MAX_NUM];
58 Error *errp;
59};
60
61static struct spapr_phb_pci_nvgpu_slot *
62spapr_nvgpu_get_slot(struct spapr_phb_pci_nvgpu_config *nvgpus, uint64_t tgt)
63{
64 int i;
65
66
67 for (i = 0; i < nvgpus->num; ++i) {
68 if (nvgpus->slots[i].tgt == tgt) {
69 return &nvgpus->slots[i];
70 }
71 }
72
73 if (nvgpus->num == ARRAY_SIZE(nvgpus->slots)) {
74 return NULL;
75 }
76
77 i = nvgpus->num;
78 nvgpus->slots[i].tgt = tgt;
79 ++nvgpus->num;
80
81 return &nvgpus->slots[i];
82}
83
84static void spapr_pci_collect_nvgpu(struct spapr_phb_pci_nvgpu_config *nvgpus,
85 PCIDevice *pdev, uint64_t tgt,
86 MemoryRegion *mr, Error **errp)
87{
88 MachineState *machine = MACHINE(qdev_get_machine());
89 SpaprMachineState *spapr = SPAPR_MACHINE(machine);
90 struct spapr_phb_pci_nvgpu_slot *nvslot = spapr_nvgpu_get_slot(nvgpus, tgt);
91
92 if (!nvslot) {
93 error_setg(errp, "Found too many GPUs per vPHB");
94 return;
95 }
96 g_assert(!nvslot->gpdev);
97 nvslot->gpdev = pdev;
98
99 nvslot->gpa = nvgpus->nv2_ram_current;
100 nvgpus->nv2_ram_current += memory_region_size(mr);
101 nvslot->numa_id = spapr->gpu_numa_id;
102 ++spapr->gpu_numa_id;
103}
104
105static void spapr_pci_collect_nvnpu(struct spapr_phb_pci_nvgpu_config *nvgpus,
106 PCIDevice *pdev, uint64_t tgt,
107 MemoryRegion *mr, Error **errp)
108{
109 struct spapr_phb_pci_nvgpu_slot *nvslot = spapr_nvgpu_get_slot(nvgpus, tgt);
110 int j;
111
112 if (!nvslot) {
113 error_setg(errp, "Found too many NVLink bridges per vPHB");
114 return;
115 }
116
117 j = nvslot->linknum;
118 if (j == ARRAY_SIZE(nvslot->links)) {
119 error_setg(errp, "Found too many NVLink bridges per GPU");
120 return;
121 }
122 ++nvslot->linknum;
123
124 g_assert(!nvslot->links[j].npdev);
125 nvslot->links[j].npdev = pdev;
126 nvslot->links[j].atsd_gpa = nvgpus->nv2_atsd_current;
127 nvgpus->nv2_atsd_current += memory_region_size(mr);
128 nvslot->links[j].link_speed =
129 object_property_get_uint(OBJECT(pdev), "nvlink2-link-speed", NULL);
130}
131
132static void spapr_phb_pci_collect_nvgpu(PCIBus *bus, PCIDevice *pdev,
133 void *opaque)
134{
135 PCIBus *sec_bus;
136 Object *po = OBJECT(pdev);
137 uint64_t tgt = object_property_get_uint(po, "nvlink2-tgt", NULL);
138
139 if (tgt) {
140 Error *local_err = NULL;
141 struct spapr_phb_pci_nvgpu_config *nvgpus = opaque;
142 Object *mr_gpu = object_property_get_link(po, "nvlink2-mr[0]", NULL);
143 Object *mr_npu = object_property_get_link(po, "nvlink2-atsd-mr[0]",
144 NULL);
145
146 g_assert(mr_gpu || mr_npu);
147 if (mr_gpu) {
148 spapr_pci_collect_nvgpu(nvgpus, pdev, tgt, MEMORY_REGION(mr_gpu),
149 &local_err);
150 } else {
151 spapr_pci_collect_nvnpu(nvgpus, pdev, tgt, MEMORY_REGION(mr_npu),
152 &local_err);
153 }
154 error_propagate(&nvgpus->errp, local_err);
155 }
156 if ((pci_default_read_config(pdev, PCI_HEADER_TYPE, 1) !=
157 PCI_HEADER_TYPE_BRIDGE)) {
158 return;
159 }
160
161 sec_bus = pci_bridge_get_sec_bus(PCI_BRIDGE(pdev));
162 if (!sec_bus) {
163 return;
164 }
165
166 pci_for_each_device(sec_bus, pci_bus_num(sec_bus),
167 spapr_phb_pci_collect_nvgpu, opaque);
168}
169
170void spapr_phb_nvgpu_setup(SpaprPhbState *sphb, Error **errp)
171{
172 int i, j, valid_gpu_num;
173 PCIBus *bus;
174
175
176 if (!sphb->nv2_gpa_win_addr || !sphb->nv2_atsd_win_addr) {
177 return;
178 }
179
180 sphb->nvgpus = g_new0(struct spapr_phb_pci_nvgpu_config, 1);
181 sphb->nvgpus->nv2_ram_current = sphb->nv2_gpa_win_addr;
182 sphb->nvgpus->nv2_atsd_current = sphb->nv2_atsd_win_addr;
183
184 bus = PCI_HOST_BRIDGE(sphb)->bus;
185 pci_for_each_device(bus, pci_bus_num(bus),
186 spapr_phb_pci_collect_nvgpu, sphb->nvgpus);
187
188 if (sphb->nvgpus->errp) {
189 error_propagate(errp, sphb->nvgpus->errp);
190 sphb->nvgpus->errp = NULL;
191 goto cleanup_exit;
192 }
193
194
195 for (i = 0, valid_gpu_num = 0; i < sphb->nvgpus->num; ++i) {
196 Object *nvmrobj;
197 struct spapr_phb_pci_nvgpu_slot *nvslot = &sphb->nvgpus->slots[i];
198
199 if (!nvslot->gpdev) {
200 continue;
201 }
202 nvmrobj = object_property_get_link(OBJECT(nvslot->gpdev),
203 "nvlink2-mr[0]", NULL);
204
205 if (!nvmrobj) {
206 continue;
207 }
208
209 ++valid_gpu_num;
210 memory_region_add_subregion(get_system_memory(), nvslot->gpa,
211 MEMORY_REGION(nvmrobj));
212
213 for (j = 0; j < nvslot->linknum; ++j) {
214 Object *atsdmrobj;
215
216 atsdmrobj = object_property_get_link(OBJECT(nvslot->links[j].npdev),
217 "nvlink2-atsd-mr[0]", NULL);
218 if (!atsdmrobj) {
219 continue;
220 }
221 memory_region_add_subregion(get_system_memory(),
222 nvslot->links[j].atsd_gpa,
223 MEMORY_REGION(atsdmrobj));
224 }
225 }
226
227 if (valid_gpu_num) {
228 return;
229 }
230
231cleanup_exit:
232 g_free(sphb->nvgpus);
233 sphb->nvgpus = NULL;
234}
235
236void spapr_phb_nvgpu_free(SpaprPhbState *sphb)
237{
238 int i, j;
239
240 if (!sphb->nvgpus) {
241 return;
242 }
243
244 for (i = 0; i < sphb->nvgpus->num; ++i) {
245 struct spapr_phb_pci_nvgpu_slot *nvslot = &sphb->nvgpus->slots[i];
246 Object *nv_mrobj = object_property_get_link(OBJECT(nvslot->gpdev),
247 "nvlink2-mr[0]", NULL);
248
249 if (nv_mrobj) {
250 memory_region_del_subregion(get_system_memory(),
251 MEMORY_REGION(nv_mrobj));
252 }
253 for (j = 0; j < nvslot->linknum; ++j) {
254 PCIDevice *npdev = nvslot->links[j].npdev;
255 Object *atsd_mrobj;
256 atsd_mrobj = object_property_get_link(OBJECT(npdev),
257 "nvlink2-atsd-mr[0]", NULL);
258 if (atsd_mrobj) {
259 memory_region_del_subregion(get_system_memory(),
260 MEMORY_REGION(atsd_mrobj));
261 }
262 }
263 }
264 g_free(sphb->nvgpus);
265 sphb->nvgpus = NULL;
266}
267
268void spapr_phb_nvgpu_populate_dt(SpaprPhbState *sphb, void *fdt, int bus_off,
269 Error **errp)
270{
271 int i, j, atsdnum = 0;
272 uint64_t atsd[8];
273
274 if (!sphb->nvgpus) {
275 return;
276 }
277
278 for (i = 0; (i < sphb->nvgpus->num) && (atsdnum < ARRAY_SIZE(atsd)); ++i) {
279 struct spapr_phb_pci_nvgpu_slot *nvslot = &sphb->nvgpus->slots[i];
280
281 if (!nvslot->gpdev) {
282 continue;
283 }
284 for (j = 0; j < nvslot->linknum; ++j) {
285 if (!nvslot->links[j].atsd_gpa) {
286 continue;
287 }
288
289 if (atsdnum == ARRAY_SIZE(atsd)) {
290 error_report("Only %"PRIuPTR" ATSD registers supported",
291 ARRAY_SIZE(atsd));
292 break;
293 }
294 atsd[atsdnum] = cpu_to_be64(nvslot->links[j].atsd_gpa);
295 ++atsdnum;
296 }
297 }
298
299 if (!atsdnum) {
300 error_setg(errp, "No ATSD registers found");
301 return;
302 }
303
304 if (!spapr_phb_eeh_available(sphb)) {
305
306
307
308
309
310
311
312 error_setg(errp, "ATSD requires separate vPHB per GPU IOMMU group");
313 return;
314 }
315
316 _FDT((fdt_setprop(fdt, bus_off, "ibm,mmio-atsd", atsd,
317 atsdnum * sizeof(atsd[0]))));
318}
319
320void spapr_phb_nvgpu_ram_populate_dt(SpaprPhbState *sphb, void *fdt)
321{
322 int i, j, linkidx, npuoff;
323 char *npuname;
324
325 if (!sphb->nvgpus) {
326 return;
327 }
328
329 npuname = g_strdup_printf("npuphb%d", sphb->index);
330 npuoff = fdt_add_subnode(fdt, 0, npuname);
331 _FDT(npuoff);
332 _FDT(fdt_setprop_cell(fdt, npuoff, "#address-cells", 1));
333 _FDT(fdt_setprop_cell(fdt, npuoff, "#size-cells", 0));
334
335 _FDT((fdt_setprop_string(fdt, npuoff, "compatible", "ibm,power9-npu")));
336 g_free(npuname);
337
338 for (i = 0, linkidx = 0; i < sphb->nvgpus->num; ++i) {
339 for (j = 0; j < sphb->nvgpus->slots[i].linknum; ++j) {
340 char *linkname = g_strdup_printf("link@%d", linkidx);
341 int off = fdt_add_subnode(fdt, npuoff, linkname);
342
343 _FDT(off);
344
345 _FDT((fdt_setprop_string(fdt, off, "compatible",
346 "ibm,npu-link")));
347 _FDT((fdt_setprop_cell(fdt, off, "phandle",
348 PHANDLE_NVLINK(sphb, i, j))));
349 _FDT((fdt_setprop_cell(fdt, off, "ibm,npu-link-index", linkidx)));
350 g_free(linkname);
351 ++linkidx;
352 }
353 }
354
355
356 for (i = 0; i < sphb->nvgpus->num; ++i) {
357 struct spapr_phb_pci_nvgpu_slot *nvslot = &sphb->nvgpus->slots[i];
358 Object *nv_mrobj = object_property_get_link(OBJECT(nvslot->gpdev),
359 "nvlink2-mr[0]", NULL);
360 uint32_t associativity[] = {
361 cpu_to_be32(0x4),
362 SPAPR_GPU_NUMA_ID,
363 SPAPR_GPU_NUMA_ID,
364 SPAPR_GPU_NUMA_ID,
365 cpu_to_be32(nvslot->numa_id)
366 };
367 uint64_t size = object_property_get_uint(nv_mrobj, "size", NULL);
368 uint64_t mem_reg[2] = { cpu_to_be64(nvslot->gpa), cpu_to_be64(size) };
369 char *mem_name = g_strdup_printf("memory@%"PRIx64, nvslot->gpa);
370 int off = fdt_add_subnode(fdt, 0, mem_name);
371
372 _FDT(off);
373 _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
374 _FDT((fdt_setprop(fdt, off, "reg", mem_reg, sizeof(mem_reg))));
375 _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
376 sizeof(associativity))));
377
378 _FDT((fdt_setprop_string(fdt, off, "compatible",
379 "ibm,coherent-device-memory")));
380
381 mem_reg[1] = cpu_to_be64(0);
382 _FDT((fdt_setprop(fdt, off, "linux,usable-memory", mem_reg,
383 sizeof(mem_reg))));
384 _FDT((fdt_setprop_cell(fdt, off, "phandle",
385 PHANDLE_GPURAM(sphb, i))));
386 g_free(mem_name);
387 }
388
389}
390
391void spapr_phb_nvgpu_populate_pcidev_dt(PCIDevice *dev, void *fdt, int offset,
392 SpaprPhbState *sphb)
393{
394 int i, j;
395
396 if (!sphb->nvgpus) {
397 return;
398 }
399
400 for (i = 0; i < sphb->nvgpus->num; ++i) {
401 struct spapr_phb_pci_nvgpu_slot *nvslot = &sphb->nvgpus->slots[i];
402
403
404 if (!nvslot->gpdev) {
405 continue;
406 }
407 if (dev == nvslot->gpdev) {
408 uint32_t npus[nvslot->linknum];
409
410 for (j = 0; j < nvslot->linknum; ++j) {
411 PCIDevice *npdev = nvslot->links[j].npdev;
412
413 npus[j] = cpu_to_be32(PHANDLE_PCIDEV(sphb, npdev));
414 }
415 _FDT(fdt_setprop(fdt, offset, "ibm,npu", npus,
416 j * sizeof(npus[0])));
417 _FDT((fdt_setprop_cell(fdt, offset, "phandle",
418 PHANDLE_PCIDEV(sphb, dev))));
419 continue;
420 }
421
422 for (j = 0; j < nvslot->linknum; ++j) {
423 if (dev != nvslot->links[j].npdev) {
424 continue;
425 }
426
427 _FDT((fdt_setprop_cell(fdt, offset, "phandle",
428 PHANDLE_PCIDEV(sphb, dev))));
429 _FDT(fdt_setprop_cell(fdt, offset, "ibm,gpu",
430 PHANDLE_PCIDEV(sphb, nvslot->gpdev)));
431 _FDT((fdt_setprop_cell(fdt, offset, "ibm,nvlink",
432 PHANDLE_NVLINK(sphb, i, j))));
433
434
435
436
437
438
439
440
441
442 _FDT(fdt_setprop_cell(fdt, offset, "memory-region",
443 PHANDLE_GPURAM(sphb, i)));
444 _FDT(fdt_setprop_u64(fdt, offset, "ibm,device-tgt-addr",
445 nvslot->tgt));
446 _FDT(fdt_setprop_cell(fdt, offset, "ibm,nvlink-speed",
447 nvslot->links[j].link_speed));
448 }
449 }
450}
451