1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/fs.h>
27#include <linux/pci.h>
28#include <linux/uaccess.h>
29#include <linux/vfio.h>
30#include <linux/slab.h>
31
32#include "vfio_pci_private.h"
33
34#define PCI_CFG_SPACE_SIZE 256
35
36
37#define PCI_CAP_ID_BASIC 0
38
39#define is_bar(offset) \
40 ((offset >= PCI_BASE_ADDRESS_0 && offset < PCI_BASE_ADDRESS_5 + 4) || \
41 (offset >= PCI_ROM_ADDRESS && offset < PCI_ROM_ADDRESS + 4))
42
43
44
45
46
47
48static const u8 pci_cap_length[PCI_CAP_ID_MAX + 1] = {
49 [PCI_CAP_ID_BASIC] = PCI_STD_HEADER_SIZEOF,
50 [PCI_CAP_ID_PM] = PCI_PM_SIZEOF,
51 [PCI_CAP_ID_AGP] = PCI_AGP_SIZEOF,
52 [PCI_CAP_ID_VPD] = PCI_CAP_VPD_SIZEOF,
53 [PCI_CAP_ID_SLOTID] = 0,
54 [PCI_CAP_ID_MSI] = 0xFF,
55 [PCI_CAP_ID_CHSWP] = 0,
56 [PCI_CAP_ID_PCIX] = 0xFF,
57 [PCI_CAP_ID_HT] = 0xFF,
58 [PCI_CAP_ID_VNDR] = 0xFF,
59 [PCI_CAP_ID_DBG] = 0,
60 [PCI_CAP_ID_CCRC] = 0,
61 [PCI_CAP_ID_SHPC] = 0,
62 [PCI_CAP_ID_SSVID] = 0,
63 [PCI_CAP_ID_AGP3] = 0,
64 [PCI_CAP_ID_SECDEV] = 0,
65 [PCI_CAP_ID_EXP] = 0xFF,
66 [PCI_CAP_ID_MSIX] = PCI_CAP_MSIX_SIZEOF,
67 [PCI_CAP_ID_SATA] = 0xFF,
68 [PCI_CAP_ID_AF] = PCI_CAP_AF_SIZEOF,
69};
70
71
72
73
74
75
76static const u16 pci_ext_cap_length[PCI_EXT_CAP_ID_MAX + 1] = {
77 [PCI_EXT_CAP_ID_ERR] = PCI_ERR_ROOT_COMMAND,
78 [PCI_EXT_CAP_ID_VC] = 0xFF,
79 [PCI_EXT_CAP_ID_DSN] = PCI_EXT_CAP_DSN_SIZEOF,
80 [PCI_EXT_CAP_ID_PWR] = PCI_EXT_CAP_PWR_SIZEOF,
81 [PCI_EXT_CAP_ID_RCLD] = 0,
82 [PCI_EXT_CAP_ID_RCILC] = 0,
83 [PCI_EXT_CAP_ID_RCEC] = 0,
84 [PCI_EXT_CAP_ID_MFVC] = 0xFF,
85 [PCI_EXT_CAP_ID_VC9] = 0xFF,
86 [PCI_EXT_CAP_ID_RCRB] = 0,
87 [PCI_EXT_CAP_ID_VNDR] = 0xFF,
88 [PCI_EXT_CAP_ID_CAC] = 0,
89 [PCI_EXT_CAP_ID_ACS] = 0xFF,
90 [PCI_EXT_CAP_ID_ARI] = PCI_EXT_CAP_ARI_SIZEOF,
91 [PCI_EXT_CAP_ID_ATS] = PCI_EXT_CAP_ATS_SIZEOF,
92 [PCI_EXT_CAP_ID_SRIOV] = PCI_EXT_CAP_SRIOV_SIZEOF,
93 [PCI_EXT_CAP_ID_MRIOV] = 0,
94 [PCI_EXT_CAP_ID_MCAST] = PCI_EXT_CAP_MCAST_ENDPOINT_SIZEOF,
95 [PCI_EXT_CAP_ID_PRI] = PCI_EXT_CAP_PRI_SIZEOF,
96 [PCI_EXT_CAP_ID_AMD_XXX] = 0,
97 [PCI_EXT_CAP_ID_REBAR] = 0xFF,
98 [PCI_EXT_CAP_ID_DPA] = 0xFF,
99 [PCI_EXT_CAP_ID_TPH] = 0xFF,
100 [PCI_EXT_CAP_ID_LTR] = PCI_EXT_CAP_LTR_SIZEOF,
101 [PCI_EXT_CAP_ID_SECPCI] = 0,
102 [PCI_EXT_CAP_ID_PMUX] = 0,
103 [PCI_EXT_CAP_ID_PASID] = 0,
104};
105
106
107
108
109
110
111
112
113struct perm_bits {
114 u8 *virt;
115 u8 *write;
116 int (*readfn)(struct vfio_pci_device *vdev, int pos, int count,
117 struct perm_bits *perm, int offset, __le32 *val);
118 int (*writefn)(struct vfio_pci_device *vdev, int pos, int count,
119 struct perm_bits *perm, int offset, __le32 val);
120};
121
122#define NO_VIRT 0
123#define ALL_VIRT 0xFFFFFFFFU
124#define NO_WRITE 0
125#define ALL_WRITE 0xFFFFFFFFU
126
127static int vfio_user_config_read(struct pci_dev *pdev, int offset,
128 __le32 *val, int count)
129{
130 int ret = -EINVAL;
131 u32 tmp_val = 0;
132
133 switch (count) {
134 case 1:
135 {
136 u8 tmp;
137 ret = pci_user_read_config_byte(pdev, offset, &tmp);
138 tmp_val = tmp;
139 break;
140 }
141 case 2:
142 {
143 u16 tmp;
144 ret = pci_user_read_config_word(pdev, offset, &tmp);
145 tmp_val = tmp;
146 break;
147 }
148 case 4:
149 ret = pci_user_read_config_dword(pdev, offset, &tmp_val);
150 break;
151 }
152
153 *val = cpu_to_le32(tmp_val);
154
155 return pcibios_err_to_errno(ret);
156}
157
158static int vfio_user_config_write(struct pci_dev *pdev, int offset,
159 __le32 val, int count)
160{
161 int ret = -EINVAL;
162 u32 tmp_val = le32_to_cpu(val);
163
164 switch (count) {
165 case 1:
166 ret = pci_user_write_config_byte(pdev, offset, tmp_val);
167 break;
168 case 2:
169 ret = pci_user_write_config_word(pdev, offset, tmp_val);
170 break;
171 case 4:
172 ret = pci_user_write_config_dword(pdev, offset, tmp_val);
173 break;
174 }
175
176 return pcibios_err_to_errno(ret);
177}
178
179static int vfio_default_config_read(struct vfio_pci_device *vdev, int pos,
180 int count, struct perm_bits *perm,
181 int offset, __le32 *val)
182{
183 __le32 virt = 0;
184
185 memcpy(val, vdev->vconfig + pos, count);
186
187 memcpy(&virt, perm->virt + offset, count);
188
189
190 if (cpu_to_le32(~0U >> (32 - (count * 8))) != virt) {
191 struct pci_dev *pdev = vdev->pdev;
192 __le32 phys_val = 0;
193 int ret;
194
195 ret = vfio_user_config_read(pdev, pos, &phys_val, count);
196 if (ret)
197 return ret;
198
199 *val = (phys_val & ~virt) | (*val & virt);
200 }
201
202 return count;
203}
204
205static int vfio_default_config_write(struct vfio_pci_device *vdev, int pos,
206 int count, struct perm_bits *perm,
207 int offset, __le32 val)
208{
209 __le32 virt = 0, write = 0;
210
211 memcpy(&write, perm->write + offset, count);
212
213 if (!write)
214 return count;
215
216 memcpy(&virt, perm->virt + offset, count);
217
218
219 if (write & virt) {
220 __le32 virt_val = 0;
221
222 memcpy(&virt_val, vdev->vconfig + pos, count);
223
224 virt_val &= ~(write & virt);
225 virt_val |= (val & (write & virt));
226
227 memcpy(vdev->vconfig + pos, &virt_val, count);
228 }
229
230
231 if (write & ~virt) {
232 struct pci_dev *pdev = vdev->pdev;
233 __le32 phys_val = 0;
234 int ret;
235
236 ret = vfio_user_config_read(pdev, pos, &phys_val, count);
237 if (ret)
238 return ret;
239
240 phys_val &= ~(write & ~virt);
241 phys_val |= (val & (write & ~virt));
242
243 ret = vfio_user_config_write(pdev, pos, phys_val, count);
244 if (ret)
245 return ret;
246 }
247
248 return count;
249}
250
251
252static int vfio_direct_config_read(struct vfio_pci_device *vdev, int pos,
253 int count, struct perm_bits *perm,
254 int offset, __le32 *val)
255{
256 int ret;
257
258 ret = vfio_user_config_read(vdev->pdev, pos, val, count);
259 if (ret)
260 return pcibios_err_to_errno(ret);
261
262 if (pos >= PCI_CFG_SPACE_SIZE) {
263 if (offset < 4)
264 memcpy(val, vdev->vconfig + pos, count);
265 } else if (pos >= PCI_STD_HEADER_SIZEOF) {
266 if (offset == PCI_CAP_LIST_ID && count > 1)
267 memcpy(val, vdev->vconfig + pos,
268 min(PCI_CAP_FLAGS, count));
269 else if (offset == PCI_CAP_LIST_NEXT)
270 memcpy(val, vdev->vconfig + pos, 1);
271 }
272
273 return count;
274}
275
276
277static int vfio_raw_config_write(struct vfio_pci_device *vdev, int pos,
278 int count, struct perm_bits *perm,
279 int offset, __le32 val)
280{
281 int ret;
282
283 ret = vfio_user_config_write(vdev->pdev, pos, val, count);
284 if (ret)
285 return ret;
286
287 return count;
288}
289
290static int vfio_raw_config_read(struct vfio_pci_device *vdev, int pos,
291 int count, struct perm_bits *perm,
292 int offset, __le32 *val)
293{
294 int ret;
295
296 ret = vfio_user_config_read(vdev->pdev, pos, val, count);
297 if (ret)
298 return pcibios_err_to_errno(ret);
299
300 return count;
301}
302
303
304static int vfio_virt_config_write(struct vfio_pci_device *vdev, int pos,
305 int count, struct perm_bits *perm,
306 int offset, __le32 val)
307{
308 memcpy(vdev->vconfig + pos, &val, count);
309 return count;
310}
311
312static int vfio_virt_config_read(struct vfio_pci_device *vdev, int pos,
313 int count, struct perm_bits *perm,
314 int offset, __le32 *val)
315{
316 memcpy(val, vdev->vconfig + pos, count);
317 return count;
318}
319
320
321static struct perm_bits cap_perms[PCI_CAP_ID_MAX + 1] = {
322 [0 ... PCI_CAP_ID_MAX] = { .readfn = vfio_direct_config_read }
323};
324static struct perm_bits ecap_perms[PCI_EXT_CAP_ID_MAX + 1] = {
325 [0 ... PCI_EXT_CAP_ID_MAX] = { .readfn = vfio_direct_config_read }
326};
327
328
329
330
331
332
333static struct perm_bits unassigned_perms = {
334 .readfn = vfio_raw_config_read,
335 .writefn = vfio_raw_config_write
336};
337
338static struct perm_bits virt_perms = {
339 .readfn = vfio_virt_config_read,
340 .writefn = vfio_virt_config_write
341};
342
343static void free_perm_bits(struct perm_bits *perm)
344{
345 kfree(perm->virt);
346 kfree(perm->write);
347 perm->virt = NULL;
348 perm->write = NULL;
349}
350
351static int alloc_perm_bits(struct perm_bits *perm, int size)
352{
353
354
355
356
357
358
359
360
361 size = round_up(size, 4);
362
363
364
365
366
367 perm->virt = kzalloc(size, GFP_KERNEL);
368 perm->write = kzalloc(size, GFP_KERNEL);
369 if (!perm->virt || !perm->write) {
370 free_perm_bits(perm);
371 return -ENOMEM;
372 }
373
374 perm->readfn = vfio_default_config_read;
375 perm->writefn = vfio_default_config_write;
376
377 return 0;
378}
379
380
381
382
383static inline void p_setb(struct perm_bits *p, int off, u8 virt, u8 write)
384{
385 p->virt[off] = virt;
386 p->write[off] = write;
387}
388
389
390static inline void p_setw(struct perm_bits *p, int off, u16 virt, u16 write)
391{
392 *(__le16 *)(&p->virt[off]) = cpu_to_le16(virt);
393 *(__le16 *)(&p->write[off]) = cpu_to_le16(write);
394}
395
396
397static inline void p_setd(struct perm_bits *p, int off, u32 virt, u32 write)
398{
399 *(__le32 *)(&p->virt[off]) = cpu_to_le32(virt);
400 *(__le32 *)(&p->write[off]) = cpu_to_le32(write);
401}
402
403
404
405
406
407static void vfio_bar_restore(struct vfio_pci_device *vdev)
408{
409 struct pci_dev *pdev = vdev->pdev;
410 u32 *rbar = vdev->rbar;
411 u16 cmd;
412 int i;
413
414 if (pdev->is_virtfn)
415 return;
416
417 pr_info("%s: %s reset recovery - restoring bars\n",
418 __func__, dev_name(&pdev->dev));
419
420 for (i = PCI_BASE_ADDRESS_0; i <= PCI_BASE_ADDRESS_5; i += 4, rbar++)
421 pci_user_write_config_dword(pdev, i, *rbar);
422
423 pci_user_write_config_dword(pdev, PCI_ROM_ADDRESS, *rbar);
424
425 if (vdev->nointx) {
426 pci_user_read_config_word(pdev, PCI_COMMAND, &cmd);
427 cmd |= PCI_COMMAND_INTX_DISABLE;
428 pci_user_write_config_word(pdev, PCI_COMMAND, cmd);
429 }
430}
431
432static __le32 vfio_generate_bar_flags(struct pci_dev *pdev, int bar)
433{
434 unsigned long flags = pci_resource_flags(pdev, bar);
435 u32 val;
436
437 if (flags & IORESOURCE_IO)
438 return cpu_to_le32(PCI_BASE_ADDRESS_SPACE_IO);
439
440 val = PCI_BASE_ADDRESS_SPACE_MEMORY;
441
442 if (flags & IORESOURCE_PREFETCH)
443 val |= PCI_BASE_ADDRESS_MEM_PREFETCH;
444
445 if (flags & IORESOURCE_MEM_64)
446 val |= PCI_BASE_ADDRESS_MEM_TYPE_64;
447
448 return cpu_to_le32(val);
449}
450
451
452
453
454
455static void vfio_bar_fixup(struct vfio_pci_device *vdev)
456{
457 struct pci_dev *pdev = vdev->pdev;
458 int i;
459 __le32 *bar;
460 u64 mask;
461
462 bar = (__le32 *)&vdev->vconfig[PCI_BASE_ADDRESS_0];
463
464 for (i = PCI_STD_RESOURCES; i <= PCI_STD_RESOURCE_END; i++, bar++) {
465 if (!pci_resource_start(pdev, i)) {
466 *bar = 0;
467 continue;
468 }
469
470 mask = ~(pci_resource_len(pdev, i) - 1);
471
472 *bar &= cpu_to_le32((u32)mask);
473 *bar |= vfio_generate_bar_flags(pdev, i);
474
475 if (*bar & cpu_to_le32(PCI_BASE_ADDRESS_MEM_TYPE_64)) {
476 bar++;
477 *bar &= cpu_to_le32((u32)(mask >> 32));
478 i++;
479 }
480 }
481
482 bar = (__le32 *)&vdev->vconfig[PCI_ROM_ADDRESS];
483
484
485
486
487
488
489 if (pci_resource_start(pdev, PCI_ROM_RESOURCE)) {
490 mask = ~(pci_resource_len(pdev, PCI_ROM_RESOURCE) - 1);
491 mask |= PCI_ROM_ADDRESS_ENABLE;
492 *bar &= cpu_to_le32((u32)mask);
493 } else if (pdev->resource[PCI_ROM_RESOURCE].flags &
494 IORESOURCE_ROM_SHADOW) {
495 mask = ~(0x20000 - 1);
496 mask |= PCI_ROM_ADDRESS_ENABLE;
497 *bar &= cpu_to_le32((u32)mask);
498 } else
499 *bar = 0;
500
501 vdev->bardirty = false;
502}
503
504static int vfio_basic_config_read(struct vfio_pci_device *vdev, int pos,
505 int count, struct perm_bits *perm,
506 int offset, __le32 *val)
507{
508 if (is_bar(offset))
509 vfio_bar_fixup(vdev);
510
511 count = vfio_default_config_read(vdev, pos, count, perm, offset, val);
512
513
514 if (offset == PCI_COMMAND && vdev->pdev->is_virtfn) {
515 u16 cmd = le16_to_cpu(*(__le16 *)&vdev->vconfig[PCI_COMMAND]);
516 u32 tmp_val = le32_to_cpu(*val);
517
518 tmp_val |= cmd & PCI_COMMAND_MEMORY;
519 *val = cpu_to_le32(tmp_val);
520 }
521
522 return count;
523}
524
525
526static bool vfio_need_bar_restore(struct vfio_pci_device *vdev)
527{
528 int i = 0, pos = PCI_BASE_ADDRESS_0, ret;
529 u32 bar;
530
531 for (; pos <= PCI_BASE_ADDRESS_5; i++, pos += 4) {
532 if (vdev->rbar[i]) {
533 ret = pci_user_read_config_dword(vdev->pdev, pos, &bar);
534 if (ret || vdev->rbar[i] != bar)
535 return true;
536 }
537 }
538
539 return false;
540}
541
542static int vfio_basic_config_write(struct vfio_pci_device *vdev, int pos,
543 int count, struct perm_bits *perm,
544 int offset, __le32 val)
545{
546 struct pci_dev *pdev = vdev->pdev;
547 __le16 *virt_cmd;
548 u16 new_cmd = 0;
549 int ret;
550
551 virt_cmd = (__le16 *)&vdev->vconfig[PCI_COMMAND];
552
553 if (offset == PCI_COMMAND) {
554 bool phys_mem, virt_mem, new_mem, phys_io, virt_io, new_io;
555 u16 phys_cmd;
556
557 ret = pci_user_read_config_word(pdev, PCI_COMMAND, &phys_cmd);
558 if (ret)
559 return ret;
560
561 new_cmd = le32_to_cpu(val);
562
563 phys_mem = !!(phys_cmd & PCI_COMMAND_MEMORY);
564 virt_mem = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_MEMORY);
565 new_mem = !!(new_cmd & PCI_COMMAND_MEMORY);
566
567 phys_io = !!(phys_cmd & PCI_COMMAND_IO);
568 virt_io = !!(le16_to_cpu(*virt_cmd) & PCI_COMMAND_IO);
569 new_io = !!(new_cmd & PCI_COMMAND_IO);
570
571
572
573
574
575
576
577
578
579 if ((new_mem && virt_mem && !phys_mem) ||
580 (new_io && virt_io && !phys_io) ||
581 vfio_need_bar_restore(vdev))
582 vfio_bar_restore(vdev);
583 }
584
585 count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
586 if (count < 0)
587 return count;
588
589
590
591
592
593 if (offset == PCI_COMMAND) {
594 u16 mask = PCI_COMMAND_MEMORY | PCI_COMMAND_IO;
595
596 *virt_cmd &= cpu_to_le16(~mask);
597 *virt_cmd |= cpu_to_le16(new_cmd & mask);
598 }
599
600
601 if (offset >= PCI_COMMAND && offset <= PCI_COMMAND + 1) {
602 bool virt_intx_disable;
603
604 virt_intx_disable = !!(le16_to_cpu(*virt_cmd) &
605 PCI_COMMAND_INTX_DISABLE);
606
607 if (virt_intx_disable && !vdev->virq_disabled) {
608 vdev->virq_disabled = true;
609 vfio_pci_intx_mask(vdev);
610 } else if (!virt_intx_disable && vdev->virq_disabled) {
611 vdev->virq_disabled = false;
612 vfio_pci_intx_unmask(vdev);
613 }
614 }
615
616 if (is_bar(offset))
617 vdev->bardirty = true;
618
619 return count;
620}
621
622
623static int __init init_pci_cap_basic_perm(struct perm_bits *perm)
624{
625 if (alloc_perm_bits(perm, PCI_STD_HEADER_SIZEOF))
626 return -ENOMEM;
627
628 perm->readfn = vfio_basic_config_read;
629 perm->writefn = vfio_basic_config_write;
630
631
632 p_setw(perm, PCI_VENDOR_ID, (u16)ALL_VIRT, NO_WRITE);
633 p_setw(perm, PCI_DEVICE_ID, (u16)ALL_VIRT, NO_WRITE);
634
635
636
637
638
639 p_setw(perm, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE, (u16)ALL_WRITE);
640
641
642 p_setw(perm, PCI_STATUS, PCI_STATUS_CAP_LIST, NO_WRITE);
643
644
645 p_setb(perm, PCI_CACHE_LINE_SIZE, NO_VIRT, (u8)ALL_WRITE);
646 p_setb(perm, PCI_LATENCY_TIMER, NO_VIRT, (u8)ALL_WRITE);
647 p_setb(perm, PCI_BIST, NO_VIRT, (u8)ALL_WRITE);
648
649
650 p_setd(perm, PCI_BASE_ADDRESS_0, ALL_VIRT, ALL_WRITE);
651 p_setd(perm, PCI_BASE_ADDRESS_1, ALL_VIRT, ALL_WRITE);
652 p_setd(perm, PCI_BASE_ADDRESS_2, ALL_VIRT, ALL_WRITE);
653 p_setd(perm, PCI_BASE_ADDRESS_3, ALL_VIRT, ALL_WRITE);
654 p_setd(perm, PCI_BASE_ADDRESS_4, ALL_VIRT, ALL_WRITE);
655 p_setd(perm, PCI_BASE_ADDRESS_5, ALL_VIRT, ALL_WRITE);
656 p_setd(perm, PCI_ROM_ADDRESS, ALL_VIRT, ALL_WRITE);
657
658
659 p_setb(perm, PCI_CAPABILITY_LIST, (u8)ALL_VIRT, NO_WRITE);
660
661
662 p_setb(perm, PCI_INTERRUPT_LINE, (u8)ALL_VIRT, (u8)ALL_WRITE);
663
664
665 p_setb(perm, PCI_INTERRUPT_PIN, (u8)ALL_VIRT, (u8)NO_WRITE);
666
667 return 0;
668}
669
670static int vfio_pm_config_write(struct vfio_pci_device *vdev, int pos,
671 int count, struct perm_bits *perm,
672 int offset, __le32 val)
673{
674 count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
675 if (count < 0)
676 return count;
677
678 if (offset == PCI_PM_CTRL) {
679 pci_power_t state;
680
681 switch (le32_to_cpu(val) & PCI_PM_CTRL_STATE_MASK) {
682 case 0:
683 state = PCI_D0;
684 break;
685 case 1:
686 state = PCI_D1;
687 break;
688 case 2:
689 state = PCI_D2;
690 break;
691 case 3:
692 state = PCI_D3hot;
693 break;
694 }
695
696 pci_set_power_state(vdev->pdev, state);
697 }
698
699 return count;
700}
701
702
703static int __init init_pci_cap_pm_perm(struct perm_bits *perm)
704{
705 if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_PM]))
706 return -ENOMEM;
707
708 perm->writefn = vfio_pm_config_write;
709
710
711
712
713
714 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
715
716
717
718
719
720
721 p_setd(perm, PCI_PM_CTRL, NO_VIRT, ~PCI_PM_CTRL_STATE_MASK);
722 return 0;
723}
724
725static int vfio_vpd_config_write(struct vfio_pci_device *vdev, int pos,
726 int count, struct perm_bits *perm,
727 int offset, __le32 val)
728{
729 struct pci_dev *pdev = vdev->pdev;
730 __le16 *paddr = (__le16 *)(vdev->vconfig + pos - offset + PCI_VPD_ADDR);
731 __le32 *pdata = (__le32 *)(vdev->vconfig + pos - offset + PCI_VPD_DATA);
732 u16 addr;
733 u32 data;
734
735
736
737
738
739
740 count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
741 if (count < 0 || offset > PCI_VPD_ADDR + 1 ||
742 offset + count <= PCI_VPD_ADDR + 1)
743 return count;
744
745 addr = le16_to_cpu(*paddr);
746
747 if (addr & PCI_VPD_ADDR_F) {
748 data = le32_to_cpu(*pdata);
749 if (pci_write_vpd(pdev, addr & ~PCI_VPD_ADDR_F, 4, &data) != 4)
750 return count;
751 } else {
752 data = 0;
753 if (pci_read_vpd(pdev, addr, 4, &data) < 0)
754 return count;
755 *pdata = cpu_to_le32(data);
756 }
757
758
759
760
761
762
763 addr ^= PCI_VPD_ADDR_F;
764 *paddr = cpu_to_le16(addr);
765
766 return count;
767}
768
769
770static int __init init_pci_cap_vpd_perm(struct perm_bits *perm)
771{
772 if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_VPD]))
773 return -ENOMEM;
774
775 perm->writefn = vfio_vpd_config_write;
776
777
778
779
780
781 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
782
783
784
785
786
787 p_setw(perm, PCI_VPD_ADDR, (u16)ALL_VIRT, (u16)ALL_WRITE);
788 p_setd(perm, PCI_VPD_DATA, ALL_VIRT, ALL_WRITE);
789
790 return 0;
791}
792
793
794static int __init init_pci_cap_pcix_perm(struct perm_bits *perm)
795{
796
797 if (alloc_perm_bits(perm, PCI_CAP_PCIX_SIZEOF_V2))
798 return -ENOMEM;
799
800 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
801
802 p_setw(perm, PCI_X_CMD, NO_VIRT, (u16)ALL_WRITE);
803 p_setd(perm, PCI_X_ECC_CSR, NO_VIRT, ALL_WRITE);
804 return 0;
805}
806
807static int vfio_exp_config_write(struct vfio_pci_device *vdev, int pos,
808 int count, struct perm_bits *perm,
809 int offset, __le32 val)
810{
811 __le16 *ctrl = (__le16 *)(vdev->vconfig + pos -
812 offset + PCI_EXP_DEVCTL);
813
814 count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
815 if (count < 0)
816 return count;
817
818
819
820
821
822
823
824 if (*ctrl & cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR)) {
825 u32 cap;
826 int ret;
827
828 *ctrl &= ~cpu_to_le16(PCI_EXP_DEVCTL_BCR_FLR);
829
830 ret = pci_user_read_config_dword(vdev->pdev,
831 pos - offset + PCI_EXP_DEVCAP,
832 &cap);
833
834 if (!ret && (cap & PCI_EXP_DEVCAP_FLR))
835 pci_try_reset_function(vdev->pdev);
836 }
837
838 return count;
839}
840
841
842static int __init init_pci_cap_exp_perm(struct perm_bits *perm)
843{
844
845 if (alloc_perm_bits(perm, PCI_CAP_EXP_ENDPOINT_SIZEOF_V2))
846 return -ENOMEM;
847
848 perm->writefn = vfio_exp_config_write;
849
850 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
851
852
853
854
855
856
857 p_setw(perm, PCI_EXP_DEVCTL,
858 PCI_EXP_DEVCTL_BCR_FLR, ~PCI_EXP_DEVCTL_PHANTOM);
859 p_setw(perm, PCI_EXP_DEVCTL2, NO_VIRT, ~PCI_EXP_DEVCTL2_ARI);
860 return 0;
861}
862
863static int vfio_af_config_write(struct vfio_pci_device *vdev, int pos,
864 int count, struct perm_bits *perm,
865 int offset, __le32 val)
866{
867 u8 *ctrl = vdev->vconfig + pos - offset + PCI_AF_CTRL;
868
869 count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
870 if (count < 0)
871 return count;
872
873
874
875
876
877
878
879 if (*ctrl & PCI_AF_CTRL_FLR) {
880 u8 cap;
881 int ret;
882
883 *ctrl &= ~PCI_AF_CTRL_FLR;
884
885 ret = pci_user_read_config_byte(vdev->pdev,
886 pos - offset + PCI_AF_CAP,
887 &cap);
888
889 if (!ret && (cap & PCI_AF_CAP_FLR) && (cap & PCI_AF_CAP_TP))
890 pci_try_reset_function(vdev->pdev);
891 }
892
893 return count;
894}
895
896
897static int __init init_pci_cap_af_perm(struct perm_bits *perm)
898{
899 if (alloc_perm_bits(perm, pci_cap_length[PCI_CAP_ID_AF]))
900 return -ENOMEM;
901
902 perm->writefn = vfio_af_config_write;
903
904 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
905 p_setb(perm, PCI_AF_CTRL, PCI_AF_CTRL_FLR, PCI_AF_CTRL_FLR);
906 return 0;
907}
908
909
910static int __init init_pci_ext_cap_err_perm(struct perm_bits *perm)
911{
912 u32 mask;
913
914 if (alloc_perm_bits(perm, pci_ext_cap_length[PCI_EXT_CAP_ID_ERR]))
915 return -ENOMEM;
916
917
918
919
920
921
922 p_setd(perm, 0, ALL_VIRT, NO_WRITE);
923
924
925 mask = PCI_ERR_UNC_UND |
926 PCI_ERR_UNC_DLP |
927 PCI_ERR_UNC_SURPDN |
928 PCI_ERR_UNC_POISON_TLP |
929 PCI_ERR_UNC_FCP |
930 PCI_ERR_UNC_COMP_TIME |
931 PCI_ERR_UNC_COMP_ABORT |
932 PCI_ERR_UNC_UNX_COMP |
933 PCI_ERR_UNC_RX_OVER |
934 PCI_ERR_UNC_MALF_TLP |
935 PCI_ERR_UNC_ECRC |
936 PCI_ERR_UNC_UNSUP |
937 PCI_ERR_UNC_ACSV |
938 PCI_ERR_UNC_INTN |
939 PCI_ERR_UNC_MCBTLP |
940 PCI_ERR_UNC_ATOMEG |
941 PCI_ERR_UNC_TLPPRE;
942 p_setd(perm, PCI_ERR_UNCOR_STATUS, NO_VIRT, mask);
943 p_setd(perm, PCI_ERR_UNCOR_MASK, NO_VIRT, mask);
944 p_setd(perm, PCI_ERR_UNCOR_SEVER, NO_VIRT, mask);
945
946 mask = PCI_ERR_COR_RCVR |
947 PCI_ERR_COR_BAD_TLP |
948 PCI_ERR_COR_BAD_DLLP |
949 PCI_ERR_COR_REP_ROLL |
950 PCI_ERR_COR_REP_TIMER |
951 PCI_ERR_COR_ADV_NFAT |
952 PCI_ERR_COR_INTERNAL |
953 PCI_ERR_COR_LOG_OVER;
954 p_setd(perm, PCI_ERR_COR_STATUS, NO_VIRT, mask);
955 p_setd(perm, PCI_ERR_COR_MASK, NO_VIRT, mask);
956
957 mask = PCI_ERR_CAP_ECRC_GENE |
958 PCI_ERR_CAP_ECRC_CHKE;
959 p_setd(perm, PCI_ERR_CAP, NO_VIRT, mask);
960 return 0;
961}
962
963
964static int __init init_pci_ext_cap_pwr_perm(struct perm_bits *perm)
965{
966 if (alloc_perm_bits(perm, pci_ext_cap_length[PCI_EXT_CAP_ID_PWR]))
967 return -ENOMEM;
968
969 p_setd(perm, 0, ALL_VIRT, NO_WRITE);
970
971
972 p_setb(perm, PCI_PWR_DATA, NO_VIRT, (u8)ALL_WRITE);
973 return 0;
974}
975
976
977
978
979void vfio_pci_uninit_perm_bits(void)
980{
981 free_perm_bits(&cap_perms[PCI_CAP_ID_BASIC]);
982
983 free_perm_bits(&cap_perms[PCI_CAP_ID_PM]);
984 free_perm_bits(&cap_perms[PCI_CAP_ID_VPD]);
985 free_perm_bits(&cap_perms[PCI_CAP_ID_PCIX]);
986 free_perm_bits(&cap_perms[PCI_CAP_ID_EXP]);
987 free_perm_bits(&cap_perms[PCI_CAP_ID_AF]);
988
989 free_perm_bits(&ecap_perms[PCI_EXT_CAP_ID_ERR]);
990 free_perm_bits(&ecap_perms[PCI_EXT_CAP_ID_PWR]);
991}
992
993int __init vfio_pci_init_perm_bits(void)
994{
995 int ret;
996
997
998 ret = init_pci_cap_basic_perm(&cap_perms[PCI_CAP_ID_BASIC]);
999
1000
1001 ret |= init_pci_cap_pm_perm(&cap_perms[PCI_CAP_ID_PM]);
1002 ret |= init_pci_cap_vpd_perm(&cap_perms[PCI_CAP_ID_VPD]);
1003 ret |= init_pci_cap_pcix_perm(&cap_perms[PCI_CAP_ID_PCIX]);
1004 cap_perms[PCI_CAP_ID_VNDR].writefn = vfio_raw_config_write;
1005 ret |= init_pci_cap_exp_perm(&cap_perms[PCI_CAP_ID_EXP]);
1006 ret |= init_pci_cap_af_perm(&cap_perms[PCI_CAP_ID_AF]);
1007
1008
1009 ret |= init_pci_ext_cap_err_perm(&ecap_perms[PCI_EXT_CAP_ID_ERR]);
1010 ret |= init_pci_ext_cap_pwr_perm(&ecap_perms[PCI_EXT_CAP_ID_PWR]);
1011 ecap_perms[PCI_EXT_CAP_ID_VNDR].writefn = vfio_raw_config_write;
1012
1013 if (ret)
1014 vfio_pci_uninit_perm_bits();
1015
1016 return ret;
1017}
1018
1019static int vfio_find_cap_start(struct vfio_pci_device *vdev, int pos)
1020{
1021 u8 cap;
1022 int base = (pos >= PCI_CFG_SPACE_SIZE) ? PCI_CFG_SPACE_SIZE :
1023 PCI_STD_HEADER_SIZEOF;
1024 cap = vdev->pci_config_map[pos];
1025
1026 if (cap == PCI_CAP_ID_BASIC)
1027 return 0;
1028
1029
1030 while (pos - 1 >= base && vdev->pci_config_map[pos - 1] == cap)
1031 pos--;
1032
1033 return pos;
1034}
1035
1036static int vfio_msi_config_read(struct vfio_pci_device *vdev, int pos,
1037 int count, struct perm_bits *perm,
1038 int offset, __le32 *val)
1039{
1040
1041 if (offset <= PCI_MSI_FLAGS && offset + count >= PCI_MSI_FLAGS) {
1042 __le16 *flags;
1043 int start;
1044
1045 start = vfio_find_cap_start(vdev, pos);
1046
1047 flags = (__le16 *)&vdev->vconfig[start];
1048
1049 *flags &= cpu_to_le16(~PCI_MSI_FLAGS_QMASK);
1050 *flags |= cpu_to_le16(vdev->msi_qmax << 1);
1051 }
1052
1053 return vfio_default_config_read(vdev, pos, count, perm, offset, val);
1054}
1055
1056static int vfio_msi_config_write(struct vfio_pci_device *vdev, int pos,
1057 int count, struct perm_bits *perm,
1058 int offset, __le32 val)
1059{
1060 count = vfio_default_config_write(vdev, pos, count, perm, offset, val);
1061 if (count < 0)
1062 return count;
1063
1064
1065 if (offset <= PCI_MSI_FLAGS && offset + count >= PCI_MSI_FLAGS) {
1066 __le16 *pflags;
1067 u16 flags;
1068 int start, ret;
1069
1070 start = vfio_find_cap_start(vdev, pos);
1071
1072 pflags = (__le16 *)&vdev->vconfig[start + PCI_MSI_FLAGS];
1073
1074 flags = le16_to_cpu(*pflags);
1075
1076
1077 if (!is_msi(vdev))
1078 flags &= ~PCI_MSI_FLAGS_ENABLE;
1079
1080
1081 if ((flags & PCI_MSI_FLAGS_QSIZE) >> 4 > vdev->msi_qmax) {
1082 flags &= ~PCI_MSI_FLAGS_QSIZE;
1083 flags |= vdev->msi_qmax << 4;
1084 }
1085
1086
1087 *pflags = cpu_to_le16(flags);
1088 ret = pci_user_write_config_word(vdev->pdev,
1089 start + PCI_MSI_FLAGS,
1090 flags);
1091 if (ret)
1092 return pcibios_err_to_errno(ret);
1093 }
1094
1095 return count;
1096}
1097
1098
1099
1100
1101
1102static int init_pci_cap_msi_perm(struct perm_bits *perm, int len, u16 flags)
1103{
1104 if (alloc_perm_bits(perm, len))
1105 return -ENOMEM;
1106
1107 perm->readfn = vfio_msi_config_read;
1108 perm->writefn = vfio_msi_config_write;
1109
1110 p_setb(perm, PCI_CAP_LIST_NEXT, (u8)ALL_VIRT, NO_WRITE);
1111
1112
1113
1114
1115
1116 p_setb(perm, PCI_MSI_FLAGS, (u8)ALL_VIRT, (u8)ALL_WRITE);
1117 p_setd(perm, PCI_MSI_ADDRESS_LO, ALL_VIRT, ALL_WRITE);
1118 if (flags & PCI_MSI_FLAGS_64BIT) {
1119 p_setd(perm, PCI_MSI_ADDRESS_HI, ALL_VIRT, ALL_WRITE);
1120 p_setw(perm, PCI_MSI_DATA_64, (u16)ALL_VIRT, (u16)ALL_WRITE);
1121 if (flags & PCI_MSI_FLAGS_MASKBIT) {
1122 p_setd(perm, PCI_MSI_MASK_64, NO_VIRT, ALL_WRITE);
1123 p_setd(perm, PCI_MSI_PENDING_64, NO_VIRT, ALL_WRITE);
1124 }
1125 } else {
1126 p_setw(perm, PCI_MSI_DATA_32, (u16)ALL_VIRT, (u16)ALL_WRITE);
1127 if (flags & PCI_MSI_FLAGS_MASKBIT) {
1128 p_setd(perm, PCI_MSI_MASK_32, NO_VIRT, ALL_WRITE);
1129 p_setd(perm, PCI_MSI_PENDING_32, NO_VIRT, ALL_WRITE);
1130 }
1131 }
1132 return 0;
1133}
1134
1135
1136static int vfio_msi_cap_len(struct vfio_pci_device *vdev, u8 pos)
1137{
1138 struct pci_dev *pdev = vdev->pdev;
1139 int len, ret;
1140 u16 flags;
1141
1142 ret = pci_read_config_word(pdev, pos + PCI_MSI_FLAGS, &flags);
1143 if (ret)
1144 return pcibios_err_to_errno(ret);
1145
1146 len = 10;
1147 if (flags & PCI_MSI_FLAGS_64BIT)
1148 len += 4;
1149 if (flags & PCI_MSI_FLAGS_MASKBIT)
1150 len += 10;
1151
1152 if (vdev->msi_perm)
1153 return len;
1154
1155 vdev->msi_perm = kmalloc(sizeof(struct perm_bits), GFP_KERNEL);
1156 if (!vdev->msi_perm)
1157 return -ENOMEM;
1158
1159 ret = init_pci_cap_msi_perm(vdev->msi_perm, len, flags);
1160 if (ret)
1161 return ret;
1162
1163 return len;
1164}
1165
1166
1167static int vfio_vc_cap_len(struct vfio_pci_device *vdev, u16 pos)
1168{
1169 struct pci_dev *pdev = vdev->pdev;
1170 u32 tmp;
1171 int ret, evcc, phases, vc_arb;
1172 int len = PCI_CAP_VC_BASE_SIZEOF;
1173
1174 ret = pci_read_config_dword(pdev, pos + PCI_VC_PORT_CAP1, &tmp);
1175 if (ret)
1176 return pcibios_err_to_errno(ret);
1177
1178 evcc = tmp & PCI_VC_CAP1_EVCC;
1179 ret = pci_read_config_dword(pdev, pos + PCI_VC_PORT_CAP2, &tmp);
1180 if (ret)
1181 return pcibios_err_to_errno(ret);
1182
1183 if (tmp & PCI_VC_CAP2_128_PHASE)
1184 phases = 128;
1185 else if (tmp & PCI_VC_CAP2_64_PHASE)
1186 phases = 64;
1187 else if (tmp & PCI_VC_CAP2_32_PHASE)
1188 phases = 32;
1189 else
1190 phases = 0;
1191
1192 vc_arb = phases * 4;
1193
1194
1195
1196
1197
1198
1199
1200 len += (1 + evcc) * PCI_CAP_VC_PER_VC_SIZEOF;
1201 if (vc_arb) {
1202 len = round_up(len, 16);
1203 len += vc_arb / 8;
1204 }
1205 return len;
1206}
1207
1208static int vfio_cap_len(struct vfio_pci_device *vdev, u8 cap, u8 pos)
1209{
1210 struct pci_dev *pdev = vdev->pdev;
1211 u32 dword;
1212 u16 word;
1213 u8 byte;
1214 int ret;
1215
1216 switch (cap) {
1217 case PCI_CAP_ID_MSI:
1218 return vfio_msi_cap_len(vdev, pos);
1219 case PCI_CAP_ID_PCIX:
1220 ret = pci_read_config_word(pdev, pos + PCI_X_CMD, &word);
1221 if (ret)
1222 return pcibios_err_to_errno(ret);
1223
1224 if (PCI_X_CMD_VERSION(word)) {
1225 if (pdev->cfg_size > PCI_CFG_SPACE_SIZE) {
1226
1227 pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE,
1228 &dword);
1229 vdev->extended_caps = (dword != 0);
1230 }
1231 return PCI_CAP_PCIX_SIZEOF_V2;
1232 } else
1233 return PCI_CAP_PCIX_SIZEOF_V0;
1234 case PCI_CAP_ID_VNDR:
1235
1236 ret = pci_read_config_byte(pdev, pos + PCI_CAP_FLAGS, &byte);
1237 if (ret)
1238 return pcibios_err_to_errno(ret);
1239
1240 return byte;
1241 case PCI_CAP_ID_EXP:
1242 if (pdev->cfg_size > PCI_CFG_SPACE_SIZE) {
1243
1244 pci_read_config_dword(pdev, PCI_CFG_SPACE_SIZE, &dword);
1245 vdev->extended_caps = (dword != 0);
1246 }
1247
1248
1249 if ((pcie_caps_reg(pdev) & PCI_EXP_FLAGS_VERS) == 1)
1250 return PCI_CAP_EXP_ENDPOINT_SIZEOF_V1;
1251 else
1252 return PCI_CAP_EXP_ENDPOINT_SIZEOF_V2;
1253 case PCI_CAP_ID_HT:
1254 ret = pci_read_config_byte(pdev, pos + 3, &byte);
1255 if (ret)
1256 return pcibios_err_to_errno(ret);
1257
1258 return (byte & HT_3BIT_CAP_MASK) ?
1259 HT_CAP_SIZEOF_SHORT : HT_CAP_SIZEOF_LONG;
1260 case PCI_CAP_ID_SATA:
1261 ret = pci_read_config_byte(pdev, pos + PCI_SATA_REGS, &byte);
1262 if (ret)
1263 return pcibios_err_to_errno(ret);
1264
1265 byte &= PCI_SATA_REGS_MASK;
1266 if (byte == PCI_SATA_REGS_INLINE)
1267 return PCI_SATA_SIZEOF_LONG;
1268 else
1269 return PCI_SATA_SIZEOF_SHORT;
1270 default:
1271 pr_warn("%s: %s unknown length for pci cap 0x%x@0x%x\n",
1272 dev_name(&pdev->dev), __func__, cap, pos);
1273 }
1274
1275 return 0;
1276}
1277
1278static int vfio_ext_cap_len(struct vfio_pci_device *vdev, u16 ecap, u16 epos)
1279{
1280 struct pci_dev *pdev = vdev->pdev;
1281 u8 byte;
1282 u32 dword;
1283 int ret;
1284
1285 switch (ecap) {
1286 case PCI_EXT_CAP_ID_VNDR:
1287 ret = pci_read_config_dword(pdev, epos + PCI_VSEC_HDR, &dword);
1288 if (ret)
1289 return pcibios_err_to_errno(ret);
1290
1291 return dword >> PCI_VSEC_HDR_LEN_SHIFT;
1292 case PCI_EXT_CAP_ID_VC:
1293 case PCI_EXT_CAP_ID_VC9:
1294 case PCI_EXT_CAP_ID_MFVC:
1295 return vfio_vc_cap_len(vdev, epos);
1296 case PCI_EXT_CAP_ID_ACS:
1297 ret = pci_read_config_byte(pdev, epos + PCI_ACS_CAP, &byte);
1298 if (ret)
1299 return pcibios_err_to_errno(ret);
1300
1301 if (byte & PCI_ACS_EC) {
1302 int bits;
1303
1304 ret = pci_read_config_byte(pdev,
1305 epos + PCI_ACS_EGRESS_BITS,
1306 &byte);
1307 if (ret)
1308 return pcibios_err_to_errno(ret);
1309
1310 bits = byte ? round_up(byte, 32) : 256;
1311 return 8 + (bits / 8);
1312 }
1313 return 8;
1314
1315 case PCI_EXT_CAP_ID_REBAR:
1316 ret = pci_read_config_byte(pdev, epos + PCI_REBAR_CTRL, &byte);
1317 if (ret)
1318 return pcibios_err_to_errno(ret);
1319
1320 byte &= PCI_REBAR_CTRL_NBAR_MASK;
1321 byte >>= PCI_REBAR_CTRL_NBAR_SHIFT;
1322
1323 return 4 + (byte * 8);
1324 case PCI_EXT_CAP_ID_DPA:
1325 ret = pci_read_config_byte(pdev, epos + PCI_DPA_CAP, &byte);
1326 if (ret)
1327 return pcibios_err_to_errno(ret);
1328
1329 byte &= PCI_DPA_CAP_SUBSTATE_MASK;
1330 return PCI_DPA_BASE_SIZEOF + byte + 1;
1331 case PCI_EXT_CAP_ID_TPH:
1332 ret = pci_read_config_dword(pdev, epos + PCI_TPH_CAP, &dword);
1333 if (ret)
1334 return pcibios_err_to_errno(ret);
1335
1336 if ((dword & PCI_TPH_CAP_LOC_MASK) == PCI_TPH_LOC_CAP) {
1337 int sts;
1338
1339 sts = dword & PCI_TPH_CAP_ST_MASK;
1340 sts >>= PCI_TPH_CAP_ST_SHIFT;
1341 return PCI_TPH_BASE_SIZEOF + (sts * 2) + 2;
1342 }
1343 return PCI_TPH_BASE_SIZEOF;
1344 default:
1345 pr_warn("%s: %s unknown length for pci ecap 0x%x@0x%x\n",
1346 dev_name(&pdev->dev), __func__, ecap, epos);
1347 }
1348
1349 return 0;
1350}
1351
1352static int vfio_fill_vconfig_bytes(struct vfio_pci_device *vdev,
1353 int offset, int size)
1354{
1355 struct pci_dev *pdev = vdev->pdev;
1356 int ret = 0;
1357
1358
1359
1360
1361
1362
1363 while (size) {
1364 int filled;
1365
1366 if (size >= 4 && !(offset % 4)) {
1367 __le32 *dwordp = (__le32 *)&vdev->vconfig[offset];
1368 u32 dword;
1369
1370 ret = pci_read_config_dword(pdev, offset, &dword);
1371 if (ret)
1372 return ret;
1373 *dwordp = cpu_to_le32(dword);
1374 filled = 4;
1375 } else if (size >= 2 && !(offset % 2)) {
1376 __le16 *wordp = (__le16 *)&vdev->vconfig[offset];
1377 u16 word;
1378
1379 ret = pci_read_config_word(pdev, offset, &word);
1380 if (ret)
1381 return ret;
1382 *wordp = cpu_to_le16(word);
1383 filled = 2;
1384 } else {
1385 u8 *byte = &vdev->vconfig[offset];
1386 ret = pci_read_config_byte(pdev, offset, byte);
1387 if (ret)
1388 return ret;
1389 filled = 1;
1390 }
1391
1392 offset += filled;
1393 size -= filled;
1394 }
1395
1396 return ret;
1397}
1398
1399static int vfio_cap_init(struct vfio_pci_device *vdev)
1400{
1401 struct pci_dev *pdev = vdev->pdev;
1402 u8 *map = vdev->pci_config_map;
1403 u16 status;
1404 u8 pos, *prev, cap;
1405 int loops, ret, caps = 0;
1406
1407
1408 ret = pci_read_config_word(pdev, PCI_STATUS, &status);
1409 if (ret)
1410 return ret;
1411
1412 if (!(status & PCI_STATUS_CAP_LIST))
1413 return 0;
1414
1415 ret = pci_read_config_byte(pdev, PCI_CAPABILITY_LIST, &pos);
1416 if (ret)
1417 return ret;
1418
1419
1420 prev = &vdev->vconfig[PCI_CAPABILITY_LIST];
1421
1422
1423 loops = (PCI_CFG_SPACE_SIZE - PCI_STD_HEADER_SIZEOF) / PCI_CAP_SIZEOF;
1424 while (pos && loops--) {
1425 u8 next;
1426 int i, len = 0;
1427
1428 ret = pci_read_config_byte(pdev, pos, &cap);
1429 if (ret)
1430 return ret;
1431
1432 ret = pci_read_config_byte(pdev,
1433 pos + PCI_CAP_LIST_NEXT, &next);
1434 if (ret)
1435 return ret;
1436
1437 if (cap <= PCI_CAP_ID_MAX) {
1438 len = pci_cap_length[cap];
1439 if (len == 0xFF) {
1440 len = vfio_cap_len(vdev, cap, pos);
1441 if (len < 0)
1442 return len;
1443 }
1444 }
1445
1446 if (!len) {
1447 pr_info("%s: %s hiding cap 0x%x\n",
1448 __func__, dev_name(&pdev->dev), cap);
1449 *prev = next;
1450 pos = next;
1451 continue;
1452 }
1453
1454
1455 for (i = 0; i < len; i++) {
1456 if (likely(map[pos + i] == PCI_CAP_ID_INVALID))
1457 continue;
1458
1459 pr_warn("%s: %s pci config conflict @0x%x, was cap 0x%x now cap 0x%x\n",
1460 __func__, dev_name(&pdev->dev),
1461 pos + i, map[pos + i], cap);
1462 }
1463
1464 BUILD_BUG_ON(PCI_CAP_ID_MAX >= PCI_CAP_ID_INVALID_VIRT);
1465
1466 memset(map + pos, cap, len);
1467 ret = vfio_fill_vconfig_bytes(vdev, pos, len);
1468 if (ret)
1469 return ret;
1470
1471 prev = &vdev->vconfig[pos + PCI_CAP_LIST_NEXT];
1472 pos = next;
1473 caps++;
1474 }
1475
1476
1477 if (!caps) {
1478 __le16 *vstatus = (__le16 *)&vdev->vconfig[PCI_STATUS];
1479 *vstatus &= ~cpu_to_le16(PCI_STATUS_CAP_LIST);
1480 }
1481
1482 return 0;
1483}
1484
1485static int vfio_ecap_init(struct vfio_pci_device *vdev)
1486{
1487 struct pci_dev *pdev = vdev->pdev;
1488 u8 *map = vdev->pci_config_map;
1489 u16 epos;
1490 __le32 *prev = NULL;
1491 int loops, ret, ecaps = 0;
1492
1493 if (!vdev->extended_caps)
1494 return 0;
1495
1496 epos = PCI_CFG_SPACE_SIZE;
1497
1498 loops = (pdev->cfg_size - PCI_CFG_SPACE_SIZE) / PCI_CAP_SIZEOF;
1499
1500 while (loops-- && epos >= PCI_CFG_SPACE_SIZE) {
1501 u32 header;
1502 u16 ecap;
1503 int i, len = 0;
1504 bool hidden = false;
1505
1506 ret = pci_read_config_dword(pdev, epos, &header);
1507 if (ret)
1508 return ret;
1509
1510 ecap = PCI_EXT_CAP_ID(header);
1511
1512 if (ecap <= PCI_EXT_CAP_ID_MAX) {
1513 len = pci_ext_cap_length[ecap];
1514 if (len == 0xFF) {
1515 len = vfio_ext_cap_len(vdev, ecap, epos);
1516 if (len < 0)
1517 return ret;
1518 }
1519 }
1520
1521 if (!len) {
1522 pr_info("%s: %s hiding ecap 0x%x@0x%x\n",
1523 __func__, dev_name(&pdev->dev), ecap, epos);
1524
1525
1526 if (prev) {
1527 u32 val = epos = PCI_EXT_CAP_NEXT(header);
1528 *prev &= cpu_to_le32(~(0xffcU << 20));
1529 *prev |= cpu_to_le32(val << 20);
1530 continue;
1531 }
1532
1533
1534
1535
1536
1537 len = PCI_CAP_SIZEOF;
1538 hidden = true;
1539 }
1540
1541 for (i = 0; i < len; i++) {
1542 if (likely(map[epos + i] == PCI_CAP_ID_INVALID))
1543 continue;
1544
1545 pr_warn("%s: %s pci config conflict @0x%x, was ecap 0x%x now ecap 0x%x\n",
1546 __func__, dev_name(&pdev->dev),
1547 epos + i, map[epos + i], ecap);
1548 }
1549
1550
1551
1552
1553
1554
1555 BUILD_BUG_ON(PCI_EXT_CAP_ID_MAX >= PCI_CAP_ID_INVALID_VIRT);
1556
1557 memset(map + epos, ecap, len);
1558 ret = vfio_fill_vconfig_bytes(vdev, epos, len);
1559 if (ret)
1560 return ret;
1561
1562
1563
1564
1565
1566
1567
1568 if (hidden)
1569 *(__le32 *)&vdev->vconfig[epos] &=
1570 cpu_to_le32((0xffcU << 20));
1571 else
1572 ecaps++;
1573
1574 prev = (__le32 *)&vdev->vconfig[epos];
1575 epos = PCI_EXT_CAP_NEXT(header);
1576 }
1577
1578 if (!ecaps)
1579 *(u32 *)&vdev->vconfig[PCI_CFG_SPACE_SIZE] = 0;
1580
1581 return 0;
1582}
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597int vfio_config_init(struct vfio_pci_device *vdev)
1598{
1599 struct pci_dev *pdev = vdev->pdev;
1600 u8 *map, *vconfig;
1601 int ret;
1602
1603
1604
1605
1606
1607
1608
1609 map = kmalloc(pdev->cfg_size, GFP_KERNEL);
1610 if (!map)
1611 return -ENOMEM;
1612
1613 vconfig = kmalloc(pdev->cfg_size, GFP_KERNEL);
1614 if (!vconfig) {
1615 kfree(map);
1616 return -ENOMEM;
1617 }
1618
1619 vdev->pci_config_map = map;
1620 vdev->vconfig = vconfig;
1621
1622 memset(map, PCI_CAP_ID_BASIC, PCI_STD_HEADER_SIZEOF);
1623 memset(map + PCI_STD_HEADER_SIZEOF, PCI_CAP_ID_INVALID,
1624 pdev->cfg_size - PCI_STD_HEADER_SIZEOF);
1625
1626 ret = vfio_fill_vconfig_bytes(vdev, 0, PCI_STD_HEADER_SIZEOF);
1627 if (ret)
1628 goto out;
1629
1630 vdev->bardirty = true;
1631
1632
1633
1634
1635
1636
1637
1638 vdev->rbar[0] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_0]);
1639 vdev->rbar[1] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_1]);
1640 vdev->rbar[2] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_2]);
1641 vdev->rbar[3] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_3]);
1642 vdev->rbar[4] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_4]);
1643 vdev->rbar[5] = le32_to_cpu(*(__le32 *)&vconfig[PCI_BASE_ADDRESS_5]);
1644 vdev->rbar[6] = le32_to_cpu(*(__le32 *)&vconfig[PCI_ROM_ADDRESS]);
1645
1646 if (pdev->is_virtfn) {
1647 *(__le16 *)&vconfig[PCI_VENDOR_ID] = cpu_to_le16(pdev->vendor);
1648 *(__le16 *)&vconfig[PCI_DEVICE_ID] = cpu_to_le16(pdev->device);
1649 }
1650
1651 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || vdev->nointx)
1652 vconfig[PCI_INTERRUPT_PIN] = 0;
1653
1654 ret = vfio_cap_init(vdev);
1655 if (ret)
1656 goto out;
1657
1658 ret = vfio_ecap_init(vdev);
1659 if (ret)
1660 goto out;
1661
1662 return 0;
1663
1664out:
1665 kfree(map);
1666 vdev->pci_config_map = NULL;
1667 kfree(vconfig);
1668 vdev->vconfig = NULL;
1669 return pcibios_err_to_errno(ret);
1670}
1671
1672void vfio_config_free(struct vfio_pci_device *vdev)
1673{
1674 kfree(vdev->vconfig);
1675 vdev->vconfig = NULL;
1676 kfree(vdev->pci_config_map);
1677 vdev->pci_config_map = NULL;
1678 kfree(vdev->msi_perm);
1679 vdev->msi_perm = NULL;
1680}
1681
1682
1683
1684
1685
1686static size_t vfio_pci_cap_remaining_dword(struct vfio_pci_device *vdev,
1687 loff_t pos)
1688{
1689 u8 cap = vdev->pci_config_map[pos];
1690 size_t i;
1691
1692 for (i = 1; (pos + i) % 4 && vdev->pci_config_map[pos + i] == cap; i++)
1693 ;
1694
1695 return i;
1696}
1697
1698static ssize_t vfio_config_do_rw(struct vfio_pci_device *vdev, char __user *buf,
1699 size_t count, loff_t *ppos, bool iswrite)
1700{
1701 struct pci_dev *pdev = vdev->pdev;
1702 struct perm_bits *perm;
1703 __le32 val = 0;
1704 int cap_start = 0, offset;
1705 u8 cap_id;
1706 ssize_t ret;
1707
1708 if (*ppos < 0 || *ppos >= pdev->cfg_size ||
1709 *ppos + count > pdev->cfg_size)
1710 return -EFAULT;
1711
1712
1713
1714
1715
1716 count = min(count, vfio_pci_cap_remaining_dword(vdev, *ppos));
1717 if (count >= 4 && !(*ppos % 4))
1718 count = 4;
1719 else if (count >= 2 && !(*ppos % 2))
1720 count = 2;
1721 else
1722 count = 1;
1723
1724 ret = count;
1725
1726 cap_id = vdev->pci_config_map[*ppos];
1727
1728 if (cap_id == PCI_CAP_ID_INVALID) {
1729 perm = &unassigned_perms;
1730 cap_start = *ppos;
1731 } else if (cap_id == PCI_CAP_ID_INVALID_VIRT) {
1732 perm = &virt_perms;
1733 cap_start = *ppos;
1734 } else {
1735 if (*ppos >= PCI_CFG_SPACE_SIZE) {
1736 WARN_ON(cap_id > PCI_EXT_CAP_ID_MAX);
1737
1738 perm = &ecap_perms[cap_id];
1739 cap_start = vfio_find_cap_start(vdev, *ppos);
1740 } else {
1741 WARN_ON(cap_id > PCI_CAP_ID_MAX);
1742
1743 perm = &cap_perms[cap_id];
1744
1745 if (cap_id == PCI_CAP_ID_MSI)
1746 perm = vdev->msi_perm;
1747
1748 if (cap_id > PCI_CAP_ID_BASIC)
1749 cap_start = vfio_find_cap_start(vdev, *ppos);
1750 }
1751 }
1752
1753 WARN_ON(!cap_start && cap_id != PCI_CAP_ID_BASIC);
1754 WARN_ON(cap_start > *ppos);
1755
1756 offset = *ppos - cap_start;
1757
1758 if (iswrite) {
1759 if (!perm->writefn)
1760 return ret;
1761
1762 if (copy_from_user(&val, buf, count))
1763 return -EFAULT;
1764
1765 ret = perm->writefn(vdev, *ppos, count, perm, offset, val);
1766 } else {
1767 if (perm->readfn) {
1768 ret = perm->readfn(vdev, *ppos, count,
1769 perm, offset, &val);
1770 if (ret < 0)
1771 return ret;
1772 }
1773
1774 if (copy_to_user(buf, &val, count))
1775 return -EFAULT;
1776 }
1777
1778 return ret;
1779}
1780
1781ssize_t vfio_pci_config_rw(struct vfio_pci_device *vdev, char __user *buf,
1782 size_t count, loff_t *ppos, bool iswrite)
1783{
1784 size_t done = 0;
1785 int ret = 0;
1786 loff_t pos = *ppos;
1787
1788 pos &= VFIO_PCI_OFFSET_MASK;
1789
1790 while (count) {
1791 ret = vfio_config_do_rw(vdev, buf, count, &pos, iswrite);
1792 if (ret < 0)
1793 return ret;
1794
1795 count -= ret;
1796 done += ret;
1797 buf += ret;
1798 pos += ret;
1799 }
1800
1801 *ppos += done;
1802
1803 return done;
1804}
1805