1
2
3#include <linux/kernel.h>
4#include <linux/ioport.h>
5#include <linux/bitmap.h>
6#include <linux/pci.h>
7
8#include <asm/opal.h>
9
10#include "pci.h"
11
12
13#include "../../../../drivers/pci/pci.h"
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146static void pnv_pci_ioda_fixup_iov_resources(struct pci_dev *pdev)
147{
148 struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
149 struct resource *res;
150 int i;
151 resource_size_t vf_bar_sz;
152 struct pnv_iov_data *iov;
153 int mul;
154
155 iov = kzalloc(sizeof(*iov), GFP_KERNEL);
156 if (!iov)
157 goto disable_iov;
158 pdev->dev.archdata.iov_data = iov;
159 mul = phb->ioda.total_pe_num;
160
161 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
162 res = &pdev->resource[i + PCI_IOV_RESOURCES];
163 if (!res->flags || res->parent)
164 continue;
165 if (!pnv_pci_is_m64_flags(res->flags)) {
166 dev_warn(&pdev->dev, "Don't support SR-IOV with non M64 VF BAR%d: %pR. \n",
167 i, res);
168 goto disable_iov;
169 }
170
171 vf_bar_sz = pci_iov_resource_size(pdev, i + PCI_IOV_RESOURCES);
172
173
174
175
176
177
178
179
180
181
182
183 if (vf_bar_sz > (phb->ioda.m64_segsize >> 2)) {
184
185
186
187
188
189
190
191 if (vf_bar_sz < SZ_32M) {
192 pci_err(pdev, "VF BAR%d: %pR can't be mapped in single PE mode\n",
193 i, res);
194 goto disable_iov;
195 }
196
197 iov->m64_single_mode[i] = true;
198 continue;
199 }
200
201
202
203
204
205 pci_dbg(pdev, " Fixing VF BAR%d: %pR to\n", i, res);
206 res->end = res->start + vf_bar_sz * mul - 1;
207 pci_dbg(pdev, " %pR\n", res);
208
209 pci_info(pdev, "VF BAR%d: %pR (expanded to %d VFs for PE alignment)",
210 i, res, mul);
211
212 iov->need_shift = true;
213 }
214
215 return;
216
217disable_iov:
218
219 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
220 res = &pdev->resource[i + PCI_IOV_RESOURCES];
221 res->flags = 0;
222 res->end = res->start - 1;
223 }
224
225 pdev->dev.archdata.iov_data = NULL;
226 kfree(iov);
227}
228
229void pnv_pci_ioda_fixup_iov(struct pci_dev *pdev)
230{
231 if (WARN_ON(pci_dev_is_added(pdev)))
232 return;
233
234 if (pdev->is_virtfn) {
235 struct pnv_ioda_pe *pe = pnv_ioda_get_pe(pdev);
236
237
238
239
240
241
242 pe->pdev = pdev;
243 WARN_ON(!(pe->flags & PNV_IODA_PE_VF));
244 } else if (pdev->is_physfn) {
245
246
247
248
249 pnv_pci_ioda_fixup_iov_resources(pdev);
250 }
251}
252
253resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev,
254 int resno)
255{
256 resource_size_t align = pci_iov_resource_size(pdev, resno);
257 struct pnv_phb *phb = pci_bus_to_pnvhb(pdev->bus);
258 struct pnv_iov_data *iov = pnv_iov_get(pdev);
259
260
261
262
263
264
265
266 if (!iov)
267 return align;
268
269
270
271
272
273
274 if (iov->m64_single_mode[resno - PCI_IOV_RESOURCES])
275 return align;
276
277
278
279
280
281
282
283
284
285
286
287 return phb->ioda.total_pe_num * align;
288}
289
290static int pnv_pci_vf_release_m64(struct pci_dev *pdev, u16 num_vfs)
291{
292 struct pnv_iov_data *iov;
293 struct pnv_phb *phb;
294 int window_id;
295
296 phb = pci_bus_to_pnvhb(pdev->bus);
297 iov = pnv_iov_get(pdev);
298
299 for_each_set_bit(window_id, iov->used_m64_bar_mask, MAX_M64_BARS) {
300 opal_pci_phb_mmio_enable(phb->opal_id,
301 OPAL_M64_WINDOW_TYPE,
302 window_id,
303 0);
304
305 clear_bit(window_id, &phb->ioda.m64_bar_alloc);
306 }
307
308 return 0;
309}
310
311
312
313
314
315
316
317static int64_t pnv_ioda_map_m64_segmented(struct pnv_phb *phb,
318 int window_id,
319 resource_size_t start,
320 resource_size_t size)
321{
322 int64_t rc;
323
324 rc = opal_pci_set_phb_mem_window(phb->opal_id,
325 OPAL_M64_WINDOW_TYPE,
326 window_id,
327 start,
328 0,
329 size);
330 if (rc)
331 goto out;
332
333 rc = opal_pci_phb_mmio_enable(phb->opal_id,
334 OPAL_M64_WINDOW_TYPE,
335 window_id,
336 OPAL_ENABLE_M64_SPLIT);
337out:
338 if (rc)
339 pr_err("Failed to map M64 window #%d: %lld\n", window_id, rc);
340
341 return rc;
342}
343
344static int64_t pnv_ioda_map_m64_single(struct pnv_phb *phb,
345 int pe_num,
346 int window_id,
347 resource_size_t start,
348 resource_size_t size)
349{
350 int64_t rc;
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369 rc = opal_pci_map_pe_mmio_window(phb->opal_id,
370 pe_num,
371 OPAL_M64_WINDOW_TYPE,
372 window_id,
373 0);
374 if (rc)
375 goto out;
376
377
378
379
380 rc = opal_pci_set_phb_mem_window(phb->opal_id,
381 OPAL_M64_WINDOW_TYPE,
382 window_id,
383 start,
384 0,
385 size);
386 if (rc)
387 goto out;
388
389
390
391
392
393 rc = opal_pci_phb_mmio_enable(phb->opal_id,
394 OPAL_M64_WINDOW_TYPE,
395 window_id,
396 OPAL_ENABLE_M64_NON_SPLIT);
397out:
398 if (rc)
399 pr_err("Error mapping single PE BAR\n");
400
401 return rc;
402}
403
404static int pnv_pci_alloc_m64_bar(struct pnv_phb *phb, struct pnv_iov_data *iov)
405{
406 int win;
407
408 do {
409 win = find_next_zero_bit(&phb->ioda.m64_bar_alloc,
410 phb->ioda.m64_bar_idx + 1, 0);
411
412 if (win >= phb->ioda.m64_bar_idx + 1)
413 return -1;
414 } while (test_and_set_bit(win, &phb->ioda.m64_bar_alloc));
415
416 set_bit(win, iov->used_m64_bar_mask);
417
418 return win;
419}
420
421static int pnv_pci_vf_assign_m64(struct pci_dev *pdev, u16 num_vfs)
422{
423 struct pnv_iov_data *iov;
424 struct pnv_phb *phb;
425 unsigned int win;
426 struct resource *res;
427 int i, j;
428 int64_t rc;
429 resource_size_t size, start;
430 int base_pe_num;
431
432 phb = pci_bus_to_pnvhb(pdev->bus);
433 iov = pnv_iov_get(pdev);
434
435 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
436 res = &pdev->resource[i + PCI_IOV_RESOURCES];
437 if (!res->flags || !res->parent)
438 continue;
439
440
441 if (!iov->m64_single_mode[i]) {
442 win = pnv_pci_alloc_m64_bar(phb, iov);
443 if (win < 0)
444 goto m64_failed;
445
446 size = resource_size(res);
447 start = res->start;
448
449 rc = pnv_ioda_map_m64_segmented(phb, win, start, size);
450 if (rc)
451 goto m64_failed;
452
453 continue;
454 }
455
456
457 size = pci_iov_resource_size(pdev, PCI_IOV_RESOURCES + i);
458 base_pe_num = iov->vf_pe_arr[0].pe_number;
459
460 for (j = 0; j < num_vfs; j++) {
461 win = pnv_pci_alloc_m64_bar(phb, iov);
462 if (win < 0)
463 goto m64_failed;
464
465 start = res->start + size * j;
466 rc = pnv_ioda_map_m64_single(phb, win,
467 base_pe_num + j,
468 start,
469 size);
470 if (rc)
471 goto m64_failed;
472 }
473 }
474 return 0;
475
476m64_failed:
477 pnv_pci_vf_release_m64(pdev, num_vfs);
478 return -EBUSY;
479}
480
481static void pnv_ioda_release_vf_PE(struct pci_dev *pdev)
482{
483 struct pnv_phb *phb;
484 struct pnv_ioda_pe *pe, *pe_n;
485
486 phb = pci_bus_to_pnvhb(pdev->bus);
487
488 if (!pdev->is_physfn)
489 return;
490
491
492 list_for_each_entry_safe(pe, pe_n, &phb->ioda.pe_list, list) {
493 if (pe->parent_dev != pdev)
494 continue;
495
496 pnv_pci_ioda2_release_pe_dma(pe);
497
498
499 mutex_lock(&phb->ioda.pe_list_mutex);
500 list_del(&pe->list);
501 mutex_unlock(&phb->ioda.pe_list_mutex);
502
503 pnv_ioda_deconfigure_pe(phb, pe);
504
505 pnv_ioda_free_pe(pe);
506 }
507}
508
509static int pnv_pci_vf_resource_shift(struct pci_dev *dev, int offset)
510{
511 struct resource *res, res2;
512 struct pnv_iov_data *iov;
513 resource_size_t size;
514 u16 num_vfs;
515 int i;
516
517 if (!dev->is_physfn)
518 return -EINVAL;
519 iov = pnv_iov_get(dev);
520
521
522
523
524
525
526
527
528
529 num_vfs = iov->num_vfs;
530 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
531 res = &dev->resource[i + PCI_IOV_RESOURCES];
532 if (!res->flags || !res->parent)
533 continue;
534 if (iov->m64_single_mode[i])
535 continue;
536
537
538
539
540
541
542
543 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
544 res2.flags = res->flags;
545 res2.start = res->start + (size * offset);
546 res2.end = res2.start + (size * num_vfs) - 1;
547
548 if (res2.end > res->end) {
549 dev_err(&dev->dev, "VF BAR%d: %pR would extend past %pR (trying to enable %d VFs shifted by %d)\n",
550 i, &res2, res, num_vfs, offset);
551 return -EBUSY;
552 }
553 }
554
555
556
557
558
559
560
561
562
563 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) {
564 res = &dev->resource[i + PCI_IOV_RESOURCES];
565 if (!res->flags || !res->parent)
566 continue;
567 if (iov->m64_single_mode[i])
568 continue;
569
570 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES);
571 res2 = *res;
572 res->start += size * offset;
573
574 dev_info(&dev->dev, "VF BAR%d: %pR shifted to %pR (%sabling %d VFs shifted by %d)\n",
575 i, &res2, res, (offset > 0) ? "En" : "Dis",
576 num_vfs, offset);
577
578 if (offset < 0) {
579 devm_release_resource(&dev->dev, &iov->holes[i]);
580 memset(&iov->holes[i], 0, sizeof(iov->holes[i]));
581 }
582
583 pci_update_resource(dev, i + PCI_IOV_RESOURCES);
584
585 if (offset > 0) {
586 iov->holes[i].start = res2.start;
587 iov->holes[i].end = res2.start + size * offset - 1;
588 iov->holes[i].flags = IORESOURCE_BUS;
589 iov->holes[i].name = "pnv_iov_reserved";
590 devm_request_resource(&dev->dev, res->parent,
591 &iov->holes[i]);
592 }
593 }
594 return 0;
595}
596
597static void pnv_pci_sriov_disable(struct pci_dev *pdev)
598{
599 u16 num_vfs, base_pe;
600 struct pnv_iov_data *iov;
601
602 iov = pnv_iov_get(pdev);
603 num_vfs = iov->num_vfs;
604 base_pe = iov->vf_pe_arr[0].pe_number;
605
606 if (WARN_ON(!iov))
607 return;
608
609
610 pnv_ioda_release_vf_PE(pdev);
611
612
613 if (iov->need_shift)
614 pnv_pci_vf_resource_shift(pdev, -base_pe);
615
616
617 pnv_pci_vf_release_m64(pdev, num_vfs);
618}
619
620static void pnv_ioda_setup_vf_PE(struct pci_dev *pdev, u16 num_vfs)
621{
622 struct pnv_phb *phb;
623 struct pnv_ioda_pe *pe;
624 int pe_num;
625 u16 vf_index;
626 struct pnv_iov_data *iov;
627 struct pci_dn *pdn;
628
629 if (!pdev->is_physfn)
630 return;
631
632 phb = pci_bus_to_pnvhb(pdev->bus);
633 pdn = pci_get_pdn(pdev);
634 iov = pnv_iov_get(pdev);
635
636
637 for (vf_index = 0; vf_index < num_vfs; vf_index++) {
638 int vf_devfn = pci_iov_virtfn_devfn(pdev, vf_index);
639 int vf_bus = pci_iov_virtfn_bus(pdev, vf_index);
640 struct pci_dn *vf_pdn;
641
642 pe = &iov->vf_pe_arr[vf_index];
643 pe->phb = phb;
644 pe->flags = PNV_IODA_PE_VF;
645 pe->pbus = NULL;
646 pe->parent_dev = pdev;
647 pe->mve_number = -1;
648 pe->rid = (vf_bus << 8) | vf_devfn;
649
650 pe_num = pe->pe_number;
651 pe_info(pe, "VF %04d:%02d:%02d.%d associated with PE#%x\n",
652 pci_domain_nr(pdev->bus), pdev->bus->number,
653 PCI_SLOT(vf_devfn), PCI_FUNC(vf_devfn), pe_num);
654
655 if (pnv_ioda_configure_pe(phb, pe)) {
656
657 pnv_ioda_free_pe(pe);
658 pe->pdev = NULL;
659 continue;
660 }
661
662
663 mutex_lock(&phb->ioda.pe_list_mutex);
664 list_add_tail(&pe->list, &phb->ioda.pe_list);
665 mutex_unlock(&phb->ioda.pe_list_mutex);
666
667
668 list_for_each_entry(vf_pdn, &pdn->parent->child_list, list) {
669 if (vf_pdn->busno == vf_bus &&
670 vf_pdn->devfn == vf_devfn) {
671 vf_pdn->pe_number = pe_num;
672 break;
673 }
674 }
675
676 pnv_pci_ioda2_setup_dma_pe(phb, pe);
677 }
678}
679
680static int pnv_pci_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
681{
682 struct pnv_ioda_pe *base_pe;
683 struct pnv_iov_data *iov;
684 struct pnv_phb *phb;
685 int ret;
686 u16 i;
687
688 phb = pci_bus_to_pnvhb(pdev->bus);
689 iov = pnv_iov_get(pdev);
690
691
692
693
694
695
696
697
698 if (phb->type != PNV_PHB_IODA2) {
699 pci_err(pdev, "SR-IOV is not supported on this PHB\n");
700 return -ENXIO;
701 }
702
703 if (!iov) {
704 dev_info(&pdev->dev, "don't support this SRIOV device with non 64bit-prefetchable IOV BAR\n");
705 return -ENOSPC;
706 }
707
708
709 base_pe = pnv_ioda_alloc_pe(phb, num_vfs);
710 if (!base_pe) {
711 pci_err(pdev, "Unable to allocate PEs for %d VFs\n", num_vfs);
712 return -EBUSY;
713 }
714
715 iov->vf_pe_arr = base_pe;
716 iov->num_vfs = num_vfs;
717
718
719 ret = pnv_pci_vf_assign_m64(pdev, num_vfs);
720 if (ret) {
721 dev_info(&pdev->dev, "Not enough M64 window resources\n");
722 goto m64_failed;
723 }
724
725
726
727
728
729
730 if (iov->need_shift) {
731 ret = pnv_pci_vf_resource_shift(pdev, base_pe->pe_number);
732 if (ret)
733 goto shift_failed;
734 }
735
736
737 pnv_ioda_setup_vf_PE(pdev, num_vfs);
738
739 return 0;
740
741shift_failed:
742 pnv_pci_vf_release_m64(pdev, num_vfs);
743
744m64_failed:
745 for (i = 0; i < num_vfs; i++)
746 pnv_ioda_free_pe(&iov->vf_pe_arr[i]);
747
748 return ret;
749}
750
751int pnv_pcibios_sriov_disable(struct pci_dev *pdev)
752{
753 pnv_pci_sriov_disable(pdev);
754
755
756 remove_sriov_vf_pdns(pdev);
757 return 0;
758}
759
760int pnv_pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs)
761{
762
763 add_sriov_vf_pdns(pdev);
764
765 return pnv_pci_sriov_enable(pdev, num_vfs);
766}
767