1
2
3
4
5
6
7
8
9
10
11
12
13
14#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
15
16#include <linux/device.h>
17#include <linux/eventfd.h>
18#include <linux/file.h>
19#include <linux/interrupt.h>
20#include <linux/iommu.h>
21#include <linux/module.h>
22#include <linux/mutex.h>
23#include <linux/notifier.h>
24#include <linux/pci.h>
25#include <linux/pm_runtime.h>
26#include <linux/slab.h>
27#include <linux/types.h>
28#include <linux/uaccess.h>
29#include <linux/vfio.h>
30#include <linux/vgaarb.h>
31
32#include "vfio_pci_private.h"
33
34#define DRIVER_VERSION "0.2"
35#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
36#define DRIVER_DESC "VFIO PCI - User Level meta-driver"
37
38static char ids[1024] __initdata;
39module_param_string(ids, ids, sizeof(ids), 0);
40MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified");
41
42static bool nointxmask;
43module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR);
44MODULE_PARM_DESC(nointxmask,
45 "Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag.");
46
47#ifdef CONFIG_VFIO_PCI_VGA
48static bool disable_vga;
49module_param(disable_vga, bool, S_IRUGO);
50MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci");
51#endif
52
53static bool disable_idle_d3;
54module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR);
55MODULE_PARM_DESC(disable_idle_d3,
56 "Disable using the PCI D3 low power state for idle, unused devices");
57
58static DEFINE_MUTEX(driver_lock);
59
60static inline bool vfio_vga_disabled(void)
61{
62#ifdef CONFIG_VFIO_PCI_VGA
63 return disable_vga;
64#else
65 return true;
66#endif
67}
68
69
70
71
72
73
74
75
76
77static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga)
78{
79 struct vfio_pci_device *vdev = opaque;
80 struct pci_dev *tmp = NULL, *pdev = vdev->pdev;
81 unsigned char max_busnr;
82 unsigned int decodes;
83
84 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus))
85 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
86 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
87
88 max_busnr = pci_bus_max_busnr(pdev->bus);
89 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
90
91 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) {
92 if (tmp == pdev ||
93 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) ||
94 pci_is_root_bus(tmp->bus))
95 continue;
96
97 if (tmp->bus->number >= pdev->bus->number &&
98 tmp->bus->number <= max_busnr) {
99 pci_dev_put(tmp);
100 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM;
101 break;
102 }
103 }
104
105 return decodes;
106}
107
108static inline bool vfio_pci_is_vga(struct pci_dev *pdev)
109{
110 return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA;
111}
112
113static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
114{
115 struct resource *res;
116 int bar;
117 struct vfio_pci_dummy_resource *dummy_res;
118
119 INIT_LIST_HEAD(&vdev->dummy_resources_list);
120
121 for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
122 res = vdev->pdev->resource + bar;
123
124 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP))
125 goto no_mmap;
126
127 if (!(res->flags & IORESOURCE_MEM))
128 goto no_mmap;
129
130
131
132
133
134
135 if (!resource_size(res))
136 goto no_mmap;
137
138 if (resource_size(res) >= PAGE_SIZE) {
139 vdev->bar_mmap_supported[bar] = true;
140 continue;
141 }
142
143 if (!(res->start & ~PAGE_MASK)) {
144
145
146
147
148
149 dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL);
150 if (dummy_res == NULL)
151 goto no_mmap;
152
153 dummy_res->resource.name = "vfio sub-page reserved";
154 dummy_res->resource.start = res->end + 1;
155 dummy_res->resource.end = res->start + PAGE_SIZE - 1;
156 dummy_res->resource.flags = res->flags;
157 if (request_resource(res->parent,
158 &dummy_res->resource)) {
159 kfree(dummy_res);
160 goto no_mmap;
161 }
162 dummy_res->index = bar;
163 list_add(&dummy_res->res_next,
164 &vdev->dummy_resources_list);
165 vdev->bar_mmap_supported[bar] = true;
166 continue;
167 }
168
169
170
171
172
173
174
175
176no_mmap:
177 vdev->bar_mmap_supported[bar] = false;
178 }
179}
180
181static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);
182static void vfio_pci_disable(struct vfio_pci_device *vdev);
183
184
185
186
187
188
189
190
191
192
193static bool vfio_pci_nointx(struct pci_dev *pdev)
194{
195 switch (pdev->vendor) {
196 case PCI_VENDOR_ID_INTEL:
197 switch (pdev->device) {
198
199 case 0x1572:
200 case 0x1574:
201 case 0x1580 ... 0x1581:
202 case 0x1583 ... 0x158b:
203 case 0x37d0 ... 0x37d2:
204 return true;
205 default:
206 return false;
207 }
208 }
209
210 return false;
211}
212
213static int vfio_pci_enable(struct vfio_pci_device *vdev)
214{
215 struct pci_dev *pdev = vdev->pdev;
216 int ret;
217 u16 cmd;
218 u8 msix_pos;
219
220 pci_set_power_state(pdev, PCI_D0);
221
222
223 pci_clear_master(pdev);
224
225 ret = pci_enable_device(pdev);
226 if (ret)
227 return ret;
228
229
230 ret = pci_try_reset_function(pdev);
231 if (ret == -EAGAIN) {
232 pci_disable_device(pdev);
233 return ret;
234 }
235
236 vdev->reset_works = !ret;
237 pci_save_state(pdev);
238 vdev->pci_saved_state = pci_store_saved_state(pdev);
239 if (!vdev->pci_saved_state)
240 pr_debug("%s: Couldn't store %s saved state\n",
241 __func__, dev_name(&pdev->dev));
242
243 if (likely(!nointxmask)) {
244 if (vfio_pci_nointx(pdev)) {
245 dev_info(&pdev->dev, "Masking broken INTx support\n");
246 vdev->nointx = true;
247 pci_intx(pdev, 0);
248 } else
249 vdev->pci_2_3 = pci_intx_mask_supported(pdev);
250 }
251
252 pci_read_config_word(pdev, PCI_COMMAND, &cmd);
253 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
254 cmd &= ~PCI_COMMAND_INTX_DISABLE;
255 pci_write_config_word(pdev, PCI_COMMAND, cmd);
256 }
257
258 ret = vfio_config_init(vdev);
259 if (ret) {
260 kfree(vdev->pci_saved_state);
261 vdev->pci_saved_state = NULL;
262 pci_disable_device(pdev);
263 return ret;
264 }
265
266 msix_pos = pdev->msix_cap;
267 if (msix_pos) {
268 u16 flags;
269 u32 table;
270
271 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags);
272 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table);
273
274 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
275 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
276 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
277 } else
278 vdev->msix_bar = 0xFF;
279
280 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev))
281 vdev->has_vga = true;
282
283
284 if (vfio_pci_is_vga(pdev) &&
285 pdev->vendor == PCI_VENDOR_ID_INTEL &&
286 IS_ENABLED(CONFIG_VFIO_PCI_IGD)) {
287 ret = vfio_pci_igd_init(vdev);
288 if (ret) {
289 dev_warn(&vdev->pdev->dev,
290 "Failed to setup Intel IGD regions\n");
291 vfio_pci_disable(vdev);
292 return ret;
293 }
294 }
295
296 vfio_pci_probe_mmaps(vdev);
297
298 return 0;
299}
300
301static void vfio_pci_disable(struct vfio_pci_device *vdev)
302{
303 struct pci_dev *pdev = vdev->pdev;
304 struct vfio_pci_dummy_resource *dummy_res, *tmp;
305 int i, bar;
306
307
308 pci_clear_master(pdev);
309
310 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
311 VFIO_IRQ_SET_ACTION_TRIGGER,
312 vdev->irq_type, 0, 0, NULL);
313
314 vdev->virq_disabled = false;
315
316 for (i = 0; i < vdev->num_regions; i++)
317 vdev->region[i].ops->release(vdev, &vdev->region[i]);
318
319 vdev->num_regions = 0;
320 kfree(vdev->region);
321 vdev->region = NULL;
322
323 vfio_config_free(vdev);
324
325 for (bar = PCI_STD_RESOURCES; bar <= PCI_STD_RESOURCE_END; bar++) {
326 if (!vdev->barmap[bar])
327 continue;
328 pci_iounmap(pdev, vdev->barmap[bar]);
329 pci_release_selected_regions(pdev, 1 << bar);
330 vdev->barmap[bar] = NULL;
331 }
332
333 list_for_each_entry_safe(dummy_res, tmp,
334 &vdev->dummy_resources_list, res_next) {
335 list_del(&dummy_res->res_next);
336 release_resource(&dummy_res->resource);
337 kfree(dummy_res);
338 }
339
340 vdev->needs_reset = true;
341
342
343
344
345
346
347
348 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
349 pr_info("%s: Couldn't reload %s saved state\n",
350 __func__, dev_name(&pdev->dev));
351
352 if (!vdev->reset_works)
353 goto out;
354
355 pci_save_state(pdev);
356 }
357
358
359
360
361
362 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
363
364
365
366
367
368 if (vdev->reset_works && !pci_try_reset_function(pdev))
369 vdev->needs_reset = false;
370
371 pci_restore_state(pdev);
372out:
373 pci_disable_device(pdev);
374
375 vfio_pci_try_bus_reset(vdev);
376
377 if (!disable_idle_d3)
378 pci_set_power_state(pdev, PCI_D3hot);
379}
380
381static void vfio_pci_release(void *device_data)
382{
383 struct vfio_pci_device *vdev = device_data;
384
385 mutex_lock(&driver_lock);
386
387 if (!(--vdev->refcnt)) {
388 vfio_spapr_pci_eeh_release(vdev->pdev);
389 vfio_pci_disable(vdev);
390 }
391
392 mutex_unlock(&driver_lock);
393
394 module_put(THIS_MODULE);
395}
396
397static int vfio_pci_open(void *device_data)
398{
399 struct vfio_pci_device *vdev = device_data;
400 int ret = 0;
401
402 if (!try_module_get(THIS_MODULE))
403 return -ENODEV;
404
405 mutex_lock(&driver_lock);
406
407 if (!vdev->refcnt) {
408 ret = vfio_pci_enable(vdev);
409 if (ret)
410 goto error;
411
412 vfio_spapr_pci_eeh_open(vdev->pdev);
413 }
414 vdev->refcnt++;
415error:
416 mutex_unlock(&driver_lock);
417 if (ret)
418 module_put(THIS_MODULE);
419 return ret;
420}
421
422static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
423{
424 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) {
425 u8 pin;
426 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
427 if (IS_ENABLED(CONFIG_VFIO_PCI_INTX) && !vdev->nointx && pin)
428 return 1;
429
430 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) {
431 u8 pos;
432 u16 flags;
433
434 pos = vdev->pdev->msi_cap;
435 if (pos) {
436 pci_read_config_word(vdev->pdev,
437 pos + PCI_MSI_FLAGS, &flags);
438 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1);
439 }
440 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) {
441 u8 pos;
442 u16 flags;
443
444 pos = vdev->pdev->msix_cap;
445 if (pos) {
446 pci_read_config_word(vdev->pdev,
447 pos + PCI_MSIX_FLAGS, &flags);
448
449 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1;
450 }
451 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) {
452 if (pci_is_pcie(vdev->pdev))
453 return 1;
454 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) {
455 return 1;
456 }
457
458 return 0;
459}
460
461static int vfio_pci_count_devs(struct pci_dev *pdev, void *data)
462{
463 (*(int *)data)++;
464 return 0;
465}
466
467struct vfio_pci_fill_info {
468 int max;
469 int cur;
470 struct vfio_pci_dependent_device *devices;
471};
472
473static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data)
474{
475 struct vfio_pci_fill_info *fill = data;
476 struct iommu_group *iommu_group;
477
478 if (fill->cur == fill->max)
479 return -EAGAIN;
480
481 iommu_group = iommu_group_get(&pdev->dev);
482 if (!iommu_group)
483 return -EPERM;
484
485 fill->devices[fill->cur].group_id = iommu_group_id(iommu_group);
486 fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus);
487 fill->devices[fill->cur].bus = pdev->bus->number;
488 fill->devices[fill->cur].devfn = pdev->devfn;
489 fill->cur++;
490 iommu_group_put(iommu_group);
491 return 0;
492}
493
494struct vfio_pci_group_entry {
495 struct vfio_group *group;
496 int id;
497};
498
499struct vfio_pci_group_info {
500 int count;
501 struct vfio_pci_group_entry *groups;
502};
503
504static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data)
505{
506 struct vfio_pci_group_info *info = data;
507 struct iommu_group *group;
508 int id, i;
509
510 group = iommu_group_get(&pdev->dev);
511 if (!group)
512 return -EPERM;
513
514 id = iommu_group_id(group);
515
516 for (i = 0; i < info->count; i++)
517 if (info->groups[i].id == id)
518 break;
519
520 iommu_group_put(group);
521
522 return (i == info->count) ? -EINVAL : 0;
523}
524
525static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot)
526{
527 for (; pdev; pdev = pdev->bus->self)
528 if (pdev->bus == slot->bus)
529 return (pdev->slot == slot);
530 return false;
531}
532
533struct vfio_pci_walk_info {
534 int (*fn)(struct pci_dev *, void *data);
535 void *data;
536 struct pci_dev *pdev;
537 bool slot;
538 int ret;
539};
540
541static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data)
542{
543 struct vfio_pci_walk_info *walk = data;
544
545 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot))
546 walk->ret = walk->fn(pdev, walk->data);
547
548 return walk->ret;
549}
550
551static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev,
552 int (*fn)(struct pci_dev *,
553 void *data), void *data,
554 bool slot)
555{
556 struct vfio_pci_walk_info walk = {
557 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0,
558 };
559
560 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk);
561
562 return walk.ret;
563}
564
565static int msix_sparse_mmap_cap(struct vfio_pci_device *vdev,
566 struct vfio_info_cap *caps)
567{
568 struct vfio_region_info_cap_sparse_mmap *sparse;
569 size_t end, size;
570 int nr_areas = 2, i = 0, ret;
571
572 end = pci_resource_len(vdev->pdev, vdev->msix_bar);
573
574
575 if (((vdev->msix_offset & PAGE_MASK) == 0) ||
576 (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) >= end))
577 nr_areas = 1;
578
579 size = sizeof(*sparse) + (nr_areas * sizeof(*sparse->areas));
580
581 sparse = kzalloc(size, GFP_KERNEL);
582 if (!sparse)
583 return -ENOMEM;
584
585 sparse->nr_areas = nr_areas;
586
587 if (vdev->msix_offset & PAGE_MASK) {
588 sparse->areas[i].offset = 0;
589 sparse->areas[i].size = vdev->msix_offset & PAGE_MASK;
590 i++;
591 }
592
593 if (PAGE_ALIGN(vdev->msix_offset + vdev->msix_size) < end) {
594 sparse->areas[i].offset = PAGE_ALIGN(vdev->msix_offset +
595 vdev->msix_size);
596 sparse->areas[i].size = end - sparse->areas[i].offset;
597 i++;
598 }
599
600 ret = vfio_info_add_capability(caps, VFIO_REGION_INFO_CAP_SPARSE_MMAP,
601 sparse);
602 kfree(sparse);
603
604 return ret;
605}
606
607int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
608 unsigned int type, unsigned int subtype,
609 const struct vfio_pci_regops *ops,
610 size_t size, u32 flags, void *data)
611{
612 struct vfio_pci_region *region;
613
614 region = krealloc(vdev->region,
615 (vdev->num_regions + 1) * sizeof(*region),
616 GFP_KERNEL);
617 if (!region)
618 return -ENOMEM;
619
620 vdev->region = region;
621 vdev->region[vdev->num_regions].type = type;
622 vdev->region[vdev->num_regions].subtype = subtype;
623 vdev->region[vdev->num_regions].ops = ops;
624 vdev->region[vdev->num_regions].size = size;
625 vdev->region[vdev->num_regions].flags = flags;
626 vdev->region[vdev->num_regions].data = data;
627
628 vdev->num_regions++;
629
630 return 0;
631}
632
633static long vfio_pci_ioctl(void *device_data,
634 unsigned int cmd, unsigned long arg)
635{
636 struct vfio_pci_device *vdev = device_data;
637 unsigned long minsz;
638
639 if (cmd == VFIO_DEVICE_GET_INFO) {
640 struct vfio_device_info info;
641
642 minsz = offsetofend(struct vfio_device_info, num_irqs);
643
644 if (copy_from_user(&info, (void __user *)arg, minsz))
645 return -EFAULT;
646
647 if (info.argsz < minsz)
648 return -EINVAL;
649
650 info.flags = VFIO_DEVICE_FLAGS_PCI;
651
652 if (vdev->reset_works)
653 info.flags |= VFIO_DEVICE_FLAGS_RESET;
654
655 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
656 info.num_irqs = VFIO_PCI_NUM_IRQS;
657
658 return copy_to_user((void __user *)arg, &info, minsz) ?
659 -EFAULT : 0;
660
661 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
662 struct pci_dev *pdev = vdev->pdev;
663 struct vfio_region_info info;
664 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
665 int i, ret;
666
667 minsz = offsetofend(struct vfio_region_info, offset);
668
669 if (copy_from_user(&info, (void __user *)arg, minsz))
670 return -EFAULT;
671
672 if (info.argsz < minsz)
673 return -EINVAL;
674
675 switch (info.index) {
676 case VFIO_PCI_CONFIG_REGION_INDEX:
677 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
678 info.size = pdev->cfg_size;
679 info.flags = VFIO_REGION_INFO_FLAG_READ |
680 VFIO_REGION_INFO_FLAG_WRITE;
681 break;
682 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
683 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
684 info.size = pci_resource_len(pdev, info.index);
685 if (!info.size) {
686 info.flags = 0;
687 break;
688 }
689
690 info.flags = VFIO_REGION_INFO_FLAG_READ |
691 VFIO_REGION_INFO_FLAG_WRITE;
692 if (vdev->bar_mmap_supported[info.index]) {
693 info.flags |= VFIO_REGION_INFO_FLAG_MMAP;
694 if (info.index == vdev->msix_bar) {
695 ret = msix_sparse_mmap_cap(vdev, &caps);
696 if (ret)
697 return ret;
698 }
699 }
700
701 break;
702 case VFIO_PCI_ROM_REGION_INDEX:
703 {
704 void __iomem *io;
705 size_t size;
706
707 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
708 info.flags = 0;
709
710
711 info.size = pci_resource_len(pdev, info.index);
712 if (!info.size) {
713
714 if (pdev->resource[PCI_ROM_RESOURCE].flags &
715 IORESOURCE_ROM_SHADOW)
716 info.size = 0x20000;
717 else
718 break;
719 }
720
721
722 io = pci_map_rom(pdev, &size);
723 if (!io || !size) {
724 info.size = 0;
725 break;
726 }
727 pci_unmap_rom(pdev, io);
728
729 info.flags = VFIO_REGION_INFO_FLAG_READ;
730 break;
731 }
732 case VFIO_PCI_VGA_REGION_INDEX:
733 if (!vdev->has_vga)
734 return -EINVAL;
735
736 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
737 info.size = 0xc0000;
738 info.flags = VFIO_REGION_INFO_FLAG_READ |
739 VFIO_REGION_INFO_FLAG_WRITE;
740
741 break;
742 default:
743 {
744 struct vfio_region_info_cap_type cap_type;
745
746 if (info.index >=
747 VFIO_PCI_NUM_REGIONS + vdev->num_regions)
748 return -EINVAL;
749
750 i = info.index - VFIO_PCI_NUM_REGIONS;
751
752 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
753 info.size = vdev->region[i].size;
754 info.flags = vdev->region[i].flags;
755
756 cap_type.type = vdev->region[i].type;
757 cap_type.subtype = vdev->region[i].subtype;
758
759 ret = vfio_info_add_capability(&caps,
760 VFIO_REGION_INFO_CAP_TYPE,
761 &cap_type);
762 if (ret)
763 return ret;
764
765 }
766 }
767
768 if (caps.size) {
769 info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
770 if (info.argsz < sizeof(info) + caps.size) {
771 info.argsz = sizeof(info) + caps.size;
772 info.cap_offset = 0;
773 } else {
774 vfio_info_cap_shift(&caps, sizeof(info));
775 if (copy_to_user((void __user *)arg +
776 sizeof(info), caps.buf,
777 caps.size)) {
778 kfree(caps.buf);
779 return -EFAULT;
780 }
781 info.cap_offset = sizeof(info);
782 }
783
784 kfree(caps.buf);
785 }
786
787 return copy_to_user((void __user *)arg, &info, minsz) ?
788 -EFAULT : 0;
789
790 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
791 struct vfio_irq_info info;
792
793 minsz = offsetofend(struct vfio_irq_info, count);
794
795 if (copy_from_user(&info, (void __user *)arg, minsz))
796 return -EFAULT;
797
798 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
799 return -EINVAL;
800
801 switch (info.index) {
802 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX:
803 case VFIO_PCI_REQ_IRQ_INDEX:
804 break;
805 case VFIO_PCI_ERR_IRQ_INDEX:
806 if (pci_is_pcie(vdev->pdev))
807 break;
808
809 default:
810 return -EINVAL;
811 }
812
813 info.flags = VFIO_IRQ_INFO_EVENTFD;
814
815 info.count = vfio_pci_get_irq_count(vdev, info.index);
816
817 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
818 info.flags |= (VFIO_IRQ_INFO_MASKABLE |
819 VFIO_IRQ_INFO_AUTOMASKED);
820 else
821 info.flags |= VFIO_IRQ_INFO_NORESIZE;
822
823 return copy_to_user((void __user *)arg, &info, minsz) ?
824 -EFAULT : 0;
825
826 } else if (cmd == VFIO_DEVICE_SET_IRQS) {
827 struct vfio_irq_set hdr;
828 u8 *data = NULL;
829 int max, ret = 0;
830 size_t data_size = 0;
831
832 minsz = offsetofend(struct vfio_irq_set, count);
833
834 if (copy_from_user(&hdr, (void __user *)arg, minsz))
835 return -EFAULT;
836
837 max = vfio_pci_get_irq_count(vdev, hdr.index);
838
839 ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
840 VFIO_PCI_NUM_IRQS, &data_size);
841 if (ret)
842 return ret;
843
844 if (data_size) {
845 data = memdup_user((void __user *)(arg + minsz),
846 data_size);
847 if (IS_ERR(data))
848 return PTR_ERR(data);
849 }
850
851 mutex_lock(&vdev->igate);
852
853 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
854 hdr.start, hdr.count, data);
855
856 mutex_unlock(&vdev->igate);
857 kfree(data);
858
859 return ret;
860
861 } else if (cmd == VFIO_DEVICE_RESET) {
862 return vdev->reset_works ?
863 pci_try_reset_function(vdev->pdev) : -EINVAL;
864
865 } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) {
866 struct vfio_pci_hot_reset_info hdr;
867 struct vfio_pci_fill_info fill = { 0 };
868 struct vfio_pci_dependent_device *devices = NULL;
869 bool slot = false;
870 int ret = 0;
871
872 minsz = offsetofend(struct vfio_pci_hot_reset_info, count);
873
874 if (copy_from_user(&hdr, (void __user *)arg, minsz))
875 return -EFAULT;
876
877 if (hdr.argsz < minsz)
878 return -EINVAL;
879
880 hdr.flags = 0;
881
882
883 if (!pci_probe_reset_slot(vdev->pdev->slot))
884 slot = true;
885 else if (pci_probe_reset_bus(vdev->pdev->bus))
886 return -ENODEV;
887
888
889 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
890 vfio_pci_count_devs,
891 &fill.max, slot);
892 if (ret)
893 return ret;
894
895 WARN_ON(!fill.max);
896
897
898
899
900
901 if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) {
902 ret = -ENOSPC;
903 hdr.count = fill.max;
904 goto reset_info_exit;
905 }
906
907 devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL);
908 if (!devices)
909 return -ENOMEM;
910
911 fill.devices = devices;
912
913 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
914 vfio_pci_fill_devs,
915 &fill, slot);
916
917
918
919
920
921
922 if (!ret)
923 hdr.count = fill.cur;
924
925reset_info_exit:
926 if (copy_to_user((void __user *)arg, &hdr, minsz))
927 ret = -EFAULT;
928
929 if (!ret) {
930 if (copy_to_user((void __user *)(arg + minsz), devices,
931 hdr.count * sizeof(*devices)))
932 ret = -EFAULT;
933 }
934
935 kfree(devices);
936 return ret;
937
938 } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) {
939 struct vfio_pci_hot_reset hdr;
940 int32_t *group_fds;
941 struct vfio_pci_group_entry *groups;
942 struct vfio_pci_group_info info;
943 bool slot = false;
944 int i, count = 0, ret = 0;
945
946 minsz = offsetofend(struct vfio_pci_hot_reset, count);
947
948 if (copy_from_user(&hdr, (void __user *)arg, minsz))
949 return -EFAULT;
950
951 if (hdr.argsz < minsz || hdr.flags)
952 return -EINVAL;
953
954
955 if (!pci_probe_reset_slot(vdev->pdev->slot))
956 slot = true;
957 else if (pci_probe_reset_bus(vdev->pdev->bus))
958 return -ENODEV;
959
960
961
962
963
964
965
966 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
967 vfio_pci_count_devs,
968 &count, slot);
969 if (ret)
970 return ret;
971
972
973 if (!hdr.count || hdr.count > count)
974 return -EINVAL;
975
976 group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL);
977 groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL);
978 if (!group_fds || !groups) {
979 kfree(group_fds);
980 kfree(groups);
981 return -ENOMEM;
982 }
983
984 if (copy_from_user(group_fds, (void __user *)(arg + minsz),
985 hdr.count * sizeof(*group_fds))) {
986 kfree(group_fds);
987 kfree(groups);
988 return -EFAULT;
989 }
990
991
992
993
994
995
996 for (i = 0; i < hdr.count; i++) {
997 struct vfio_group *group;
998 struct fd f = fdget(group_fds[i]);
999 if (!f.file) {
1000 ret = -EBADF;
1001 break;
1002 }
1003
1004 group = vfio_group_get_external_user(f.file);
1005 fdput(f);
1006 if (IS_ERR(group)) {
1007 ret = PTR_ERR(group);
1008 break;
1009 }
1010
1011 groups[i].group = group;
1012 groups[i].id = vfio_external_user_iommu_id(group);
1013 }
1014
1015 kfree(group_fds);
1016
1017
1018 if (ret)
1019 goto hot_reset_release;
1020
1021 info.count = hdr.count;
1022 info.groups = groups;
1023
1024
1025
1026
1027
1028 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1029 vfio_pci_validate_devs,
1030 &info, slot);
1031 if (!ret)
1032
1033 ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
1034 pci_try_reset_bus(vdev->pdev->bus);
1035
1036hot_reset_release:
1037 for (i--; i >= 0; i--)
1038 vfio_group_put_external_user(groups[i].group);
1039
1040 kfree(groups);
1041 return ret;
1042 }
1043
1044 return -ENOTTY;
1045}
1046
1047static ssize_t vfio_pci_rw(void *device_data, char __user *buf,
1048 size_t count, loff_t *ppos, bool iswrite)
1049{
1050 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
1051 struct vfio_pci_device *vdev = device_data;
1052
1053 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1054 return -EINVAL;
1055
1056 switch (index) {
1057 case VFIO_PCI_CONFIG_REGION_INDEX:
1058 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1059
1060 case VFIO_PCI_ROM_REGION_INDEX:
1061 if (iswrite)
1062 return -EINVAL;
1063 return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1064
1065 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1066 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1067
1068 case VFIO_PCI_VGA_REGION_INDEX:
1069 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1070 default:
1071 index -= VFIO_PCI_NUM_REGIONS;
1072 return vdev->region[index].ops->rw(vdev, buf,
1073 count, ppos, iswrite);
1074 }
1075
1076 return -EINVAL;
1077}
1078
1079static ssize_t vfio_pci_read(void *device_data, char __user *buf,
1080 size_t count, loff_t *ppos)
1081{
1082 if (!count)
1083 return 0;
1084
1085 return vfio_pci_rw(device_data, buf, count, ppos, false);
1086}
1087
1088static ssize_t vfio_pci_write(void *device_data, const char __user *buf,
1089 size_t count, loff_t *ppos)
1090{
1091 if (!count)
1092 return 0;
1093
1094 return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true);
1095}
1096
1097static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma)
1098{
1099 struct vfio_pci_device *vdev = device_data;
1100 struct pci_dev *pdev = vdev->pdev;
1101 unsigned int index;
1102 u64 phys_len, req_len, pgoff, req_start;
1103 int ret;
1104
1105 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1106
1107 if (vma->vm_end < vma->vm_start)
1108 return -EINVAL;
1109 if ((vma->vm_flags & VM_SHARED) == 0)
1110 return -EINVAL;
1111 if (index >= VFIO_PCI_ROM_REGION_INDEX)
1112 return -EINVAL;
1113 if (!vdev->bar_mmap_supported[index])
1114 return -EINVAL;
1115
1116 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index));
1117 req_len = vma->vm_end - vma->vm_start;
1118 pgoff = vma->vm_pgoff &
1119 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
1120 req_start = pgoff << PAGE_SHIFT;
1121
1122 if (req_start + req_len > phys_len)
1123 return -EINVAL;
1124
1125 if (index == vdev->msix_bar) {
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136 if (!(req_start >= vdev->msix_offset + vdev->msix_size ||
1137 req_start + req_len <= vdev->msix_offset))
1138 return -EINVAL;
1139 }
1140
1141
1142
1143
1144
1145 if (!vdev->barmap[index]) {
1146 ret = pci_request_selected_regions(pdev,
1147 1 << index, "vfio-pci");
1148 if (ret)
1149 return ret;
1150
1151 vdev->barmap[index] = pci_iomap(pdev, index, 0);
1152 if (!vdev->barmap[index]) {
1153 pci_release_selected_regions(pdev, 1 << index);
1154 return -ENOMEM;
1155 }
1156 }
1157
1158 vma->vm_private_data = vdev;
1159 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1160 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff;
1161
1162 return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
1163 req_len, vma->vm_page_prot);
1164}
1165
1166static void vfio_pci_request(void *device_data, unsigned int count)
1167{
1168 struct vfio_pci_device *vdev = device_data;
1169
1170 mutex_lock(&vdev->igate);
1171
1172 if (vdev->req_trigger) {
1173 if (!(count % 10))
1174 dev_notice_ratelimited(&vdev->pdev->dev,
1175 "Relaying device request to user (#%u)\n",
1176 count);
1177 eventfd_signal(vdev->req_trigger, 1);
1178 } else if (count == 0) {
1179 dev_warn(&vdev->pdev->dev,
1180 "No device request channel registered, blocked until released by user\n");
1181 }
1182
1183 mutex_unlock(&vdev->igate);
1184}
1185
1186static const struct vfio_device_ops vfio_pci_ops = {
1187 .name = "vfio-pci",
1188 .open = vfio_pci_open,
1189 .release = vfio_pci_release,
1190 .ioctl = vfio_pci_ioctl,
1191 .read = vfio_pci_read,
1192 .write = vfio_pci_write,
1193 .mmap = vfio_pci_mmap,
1194 .request = vfio_pci_request,
1195};
1196
1197static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1198{
1199 struct vfio_pci_device *vdev;
1200 struct iommu_group *group;
1201 int ret;
1202
1203 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL)
1204 return -EINVAL;
1205
1206 group = vfio_iommu_group_get(&pdev->dev);
1207 if (!group)
1208 return -EINVAL;
1209
1210 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
1211 if (!vdev) {
1212 vfio_iommu_group_put(group, &pdev->dev);
1213 return -ENOMEM;
1214 }
1215
1216 vdev->pdev = pdev;
1217 vdev->irq_type = VFIO_PCI_NUM_IRQS;
1218 mutex_init(&vdev->igate);
1219 spin_lock_init(&vdev->irqlock);
1220
1221 ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
1222 if (ret) {
1223 vfio_iommu_group_put(group, &pdev->dev);
1224 kfree(vdev);
1225 return ret;
1226 }
1227
1228 if (vfio_pci_is_vga(pdev)) {
1229 vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
1230 vga_set_legacy_decoding(pdev,
1231 vfio_pci_set_vga_decode(vdev, false));
1232 }
1233
1234 if (!disable_idle_d3) {
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244 pci_set_power_state(pdev, PCI_D0);
1245 pci_set_power_state(pdev, PCI_D3hot);
1246 }
1247
1248 return ret;
1249}
1250
1251static void vfio_pci_remove(struct pci_dev *pdev)
1252{
1253 struct vfio_pci_device *vdev;
1254
1255 vdev = vfio_del_group_dev(&pdev->dev);
1256 if (!vdev)
1257 return;
1258
1259 vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev);
1260 kfree(vdev->region);
1261 kfree(vdev);
1262
1263 if (vfio_pci_is_vga(pdev)) {
1264 vga_client_register(pdev, NULL, NULL, NULL);
1265 vga_set_legacy_decoding(pdev,
1266 VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM |
1267 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM);
1268 }
1269
1270 if (!disable_idle_d3)
1271 pci_set_power_state(pdev, PCI_D0);
1272}
1273
1274static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev,
1275 pci_channel_state_t state)
1276{
1277 struct vfio_pci_device *vdev;
1278 struct vfio_device *device;
1279
1280 device = vfio_device_get_from_dev(&pdev->dev);
1281 if (device == NULL)
1282 return PCI_ERS_RESULT_DISCONNECT;
1283
1284 vdev = vfio_device_data(device);
1285 if (vdev == NULL) {
1286 vfio_device_put(device);
1287 return PCI_ERS_RESULT_DISCONNECT;
1288 }
1289
1290 mutex_lock(&vdev->igate);
1291
1292 if (vdev->err_trigger)
1293 eventfd_signal(vdev->err_trigger, 1);
1294
1295 mutex_unlock(&vdev->igate);
1296
1297 vfio_device_put(device);
1298
1299 return PCI_ERS_RESULT_CAN_RECOVER;
1300}
1301
1302static const struct pci_error_handlers vfio_err_handlers = {
1303 .error_detected = vfio_pci_aer_err_detected,
1304};
1305
1306static struct pci_driver vfio_pci_driver = {
1307 .name = "vfio-pci",
1308 .id_table = NULL,
1309 .probe = vfio_pci_probe,
1310 .remove = vfio_pci_remove,
1311 .err_handler = &vfio_err_handlers,
1312};
1313
1314struct vfio_devices {
1315 struct vfio_device **devices;
1316 int cur_index;
1317 int max_index;
1318};
1319
1320static int vfio_pci_get_devs(struct pci_dev *pdev, void *data)
1321{
1322 struct vfio_devices *devs = data;
1323 struct vfio_device *device;
1324
1325 if (devs->cur_index == devs->max_index)
1326 return -ENOSPC;
1327
1328 device = vfio_device_get_from_dev(&pdev->dev);
1329 if (!device)
1330 return -EINVAL;
1331
1332 if (pci_dev_driver(pdev) != &vfio_pci_driver) {
1333 vfio_device_put(device);
1334 return -EBUSY;
1335 }
1336
1337 devs->devices[devs->cur_index++] = device;
1338 return 0;
1339}
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
1354{
1355 struct vfio_devices devs = { .cur_index = 0 };
1356 int i = 0, ret = -EINVAL;
1357 bool needs_reset = false, slot = false;
1358 struct vfio_pci_device *tmp;
1359
1360 if (!pci_probe_reset_slot(vdev->pdev->slot))
1361 slot = true;
1362 else if (pci_probe_reset_bus(vdev->pdev->bus))
1363 return;
1364
1365 if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
1366 &i, slot) || !i)
1367 return;
1368
1369 devs.max_index = i;
1370 devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL);
1371 if (!devs.devices)
1372 return;
1373
1374 if (vfio_pci_for_each_slot_or_bus(vdev->pdev,
1375 vfio_pci_get_devs, &devs, slot))
1376 goto put_devs;
1377
1378 for (i = 0; i < devs.cur_index; i++) {
1379 tmp = vfio_device_data(devs.devices[i]);
1380 if (tmp->needs_reset)
1381 needs_reset = true;
1382 if (tmp->refcnt)
1383 goto put_devs;
1384 }
1385
1386 if (needs_reset)
1387 ret = slot ? pci_try_reset_slot(vdev->pdev->slot) :
1388 pci_try_reset_bus(vdev->pdev->bus);
1389
1390put_devs:
1391 for (i = 0; i < devs.cur_index; i++) {
1392 tmp = vfio_device_data(devs.devices[i]);
1393 if (!ret)
1394 tmp->needs_reset = false;
1395
1396 if (!tmp->refcnt && !disable_idle_d3)
1397 pci_set_power_state(tmp->pdev, PCI_D3hot);
1398
1399 vfio_device_put(devs.devices[i]);
1400 }
1401
1402 kfree(devs.devices);
1403}
1404
1405static void __exit vfio_pci_cleanup(void)
1406{
1407 pci_unregister_driver(&vfio_pci_driver);
1408 vfio_pci_uninit_perm_bits();
1409}
1410
1411static void __init vfio_pci_fill_ids(void)
1412{
1413 char *p, *id;
1414 int rc;
1415
1416
1417 if (ids[0] == '\0')
1418 return;
1419
1420
1421 p = ids;
1422 while ((id = strsep(&p, ","))) {
1423 unsigned int vendor, device, subvendor = PCI_ANY_ID,
1424 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
1425 int fields;
1426
1427 if (!strlen(id))
1428 continue;
1429
1430 fields = sscanf(id, "%x:%x:%x:%x:%x:%x",
1431 &vendor, &device, &subvendor, &subdevice,
1432 &class, &class_mask);
1433
1434 if (fields < 2) {
1435 pr_warn("invalid id string \"%s\"\n", id);
1436 continue;
1437 }
1438
1439 rc = pci_add_dynid(&vfio_pci_driver, vendor, device,
1440 subvendor, subdevice, class, class_mask, 0);
1441 if (rc)
1442 pr_warn("failed to add dynamic id [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x (%d)\n",
1443 vendor, device, subvendor, subdevice,
1444 class, class_mask, rc);
1445 else
1446 pr_info("add [%04hx:%04hx[%04hx:%04hx]] class %#08x/%08x\n",
1447 vendor, device, subvendor, subdevice,
1448 class, class_mask);
1449 }
1450}
1451
1452static int __init vfio_pci_init(void)
1453{
1454 int ret;
1455
1456
1457 ret = vfio_pci_init_perm_bits();
1458 if (ret)
1459 return ret;
1460
1461
1462 ret = pci_register_driver(&vfio_pci_driver);
1463 if (ret)
1464 goto out_driver;
1465
1466 vfio_pci_fill_ids();
1467
1468 return 0;
1469
1470out_driver:
1471 vfio_pci_uninit_perm_bits();
1472 return ret;
1473}
1474
1475module_init(vfio_pci_init);
1476module_exit(vfio_pci_cleanup);
1477
1478MODULE_VERSION(DRIVER_VERSION);
1479MODULE_LICENSE("GPL v2");
1480MODULE_AUTHOR(DRIVER_AUTHOR);
1481MODULE_DESCRIPTION(DRIVER_DESC);
1482