1
2
3
4
5
6
7#include <linux/pci.h>
8#include <linux/module.h>
9#include <linux/init.h>
10#include <linux/device.h>
11#include <linux/mempolicy.h>
12#include <linux/string.h>
13#include <linux/slab.h>
14#include <linux/sched.h>
15#include <linux/cpu.h>
16#include <linux/pm_runtime.h>
17#include <linux/suspend.h>
18#include <linux/kexec.h>
19#include <linux/of_device.h>
20#include <linux/acpi.h>
21#include "pci.h"
22#include "pcie/portdrv.h"
23
24struct pci_dynid {
25 struct list_head node;
26 struct pci_device_id id;
27};
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50int pci_add_dynid(struct pci_driver *drv,
51 unsigned int vendor, unsigned int device,
52 unsigned int subvendor, unsigned int subdevice,
53 unsigned int class, unsigned int class_mask,
54 unsigned long driver_data)
55{
56 struct pci_dynid *dynid;
57
58 dynid = kzalloc(sizeof(*dynid), GFP_KERNEL);
59 if (!dynid)
60 return -ENOMEM;
61
62 dynid->id.vendor = vendor;
63 dynid->id.device = device;
64 dynid->id.subvendor = subvendor;
65 dynid->id.subdevice = subdevice;
66 dynid->id.class = class;
67 dynid->id.class_mask = class_mask;
68 dynid->id.driver_data = driver_data;
69
70 spin_lock(&drv->dynids.lock);
71 list_add_tail(&dynid->node, &drv->dynids.list);
72 spin_unlock(&drv->dynids.lock);
73
74 return driver_attach(&drv->driver);
75}
76EXPORT_SYMBOL_GPL(pci_add_dynid);
77
78static void pci_free_dynids(struct pci_driver *drv)
79{
80 struct pci_dynid *dynid, *n;
81
82 spin_lock(&drv->dynids.lock);
83 list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) {
84 list_del(&dynid->node);
85 kfree(dynid);
86 }
87 spin_unlock(&drv->dynids.lock);
88}
89
90
91
92
93
94
95
96
97
98static ssize_t new_id_store(struct device_driver *driver, const char *buf,
99 size_t count)
100{
101 struct pci_driver *pdrv = to_pci_driver(driver);
102 const struct pci_device_id *ids = pdrv->id_table;
103 __u32 vendor, device, subvendor = PCI_ANY_ID,
104 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
105 unsigned long driver_data = 0;
106 int fields = 0;
107 int retval = 0;
108
109 fields = sscanf(buf, "%x %x %x %x %x %x %lx",
110 &vendor, &device, &subvendor, &subdevice,
111 &class, &class_mask, &driver_data);
112 if (fields < 2)
113 return -EINVAL;
114
115 if (fields != 7) {
116 struct pci_dev *pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
117 if (!pdev)
118 return -ENOMEM;
119
120 pdev->vendor = vendor;
121 pdev->device = device;
122 pdev->subsystem_vendor = subvendor;
123 pdev->subsystem_device = subdevice;
124 pdev->class = class;
125
126 if (pci_match_id(pdrv->id_table, pdev))
127 retval = -EEXIST;
128
129 kfree(pdev);
130
131 if (retval)
132 return retval;
133 }
134
135
136
137 if (ids) {
138 retval = -EINVAL;
139 while (ids->vendor || ids->subvendor || ids->class_mask) {
140 if (driver_data == ids->driver_data) {
141 retval = 0;
142 break;
143 }
144 ids++;
145 }
146 if (retval)
147 return retval;
148 }
149
150 retval = pci_add_dynid(pdrv, vendor, device, subvendor, subdevice,
151 class, class_mask, driver_data);
152 if (retval)
153 return retval;
154 return count;
155}
156static DRIVER_ATTR_WO(new_id);
157
158
159
160
161
162
163
164
165
166static ssize_t remove_id_store(struct device_driver *driver, const char *buf,
167 size_t count)
168{
169 struct pci_dynid *dynid, *n;
170 struct pci_driver *pdrv = to_pci_driver(driver);
171 __u32 vendor, device, subvendor = PCI_ANY_ID,
172 subdevice = PCI_ANY_ID, class = 0, class_mask = 0;
173 int fields = 0;
174 size_t retval = -ENODEV;
175
176 fields = sscanf(buf, "%x %x %x %x %x %x",
177 &vendor, &device, &subvendor, &subdevice,
178 &class, &class_mask);
179 if (fields < 2)
180 return -EINVAL;
181
182 spin_lock(&pdrv->dynids.lock);
183 list_for_each_entry_safe(dynid, n, &pdrv->dynids.list, node) {
184 struct pci_device_id *id = &dynid->id;
185 if ((id->vendor == vendor) &&
186 (id->device == device) &&
187 (subvendor == PCI_ANY_ID || id->subvendor == subvendor) &&
188 (subdevice == PCI_ANY_ID || id->subdevice == subdevice) &&
189 !((id->class ^ class) & class_mask)) {
190 list_del(&dynid->node);
191 kfree(dynid);
192 retval = count;
193 break;
194 }
195 }
196 spin_unlock(&pdrv->dynids.lock);
197
198 return retval;
199}
200static DRIVER_ATTR_WO(remove_id);
201
202static struct attribute *pci_drv_attrs[] = {
203 &driver_attr_new_id.attr,
204 &driver_attr_remove_id.attr,
205 NULL,
206};
207ATTRIBUTE_GROUPS(pci_drv);
208
209
210
211
212
213
214
215
216
217
218
219
220
221const struct pci_device_id *pci_match_id(const struct pci_device_id *ids,
222 struct pci_dev *dev)
223{
224 if (ids) {
225 while (ids->vendor || ids->subvendor || ids->class_mask) {
226 if (pci_match_one_device(ids, dev))
227 return ids;
228 ids++;
229 }
230 }
231 return NULL;
232}
233EXPORT_SYMBOL(pci_match_id);
234
235static const struct pci_device_id pci_device_id_any = {
236 .vendor = PCI_ANY_ID,
237 .device = PCI_ANY_ID,
238 .subvendor = PCI_ANY_ID,
239 .subdevice = PCI_ANY_ID,
240};
241
242
243
244
245
246
247
248
249
250
251static const struct pci_device_id *pci_match_device(struct pci_driver *drv,
252 struct pci_dev *dev)
253{
254 struct pci_dynid *dynid;
255 const struct pci_device_id *found_id = NULL;
256
257
258 if (dev->driver_override && strcmp(dev->driver_override, drv->name))
259 return NULL;
260
261
262 spin_lock(&drv->dynids.lock);
263 list_for_each_entry(dynid, &drv->dynids.list, node) {
264 if (pci_match_one_device(&dynid->id, dev)) {
265 found_id = &dynid->id;
266 break;
267 }
268 }
269 spin_unlock(&drv->dynids.lock);
270
271 if (!found_id)
272 found_id = pci_match_id(drv->id_table, dev);
273
274
275 if (!found_id && dev->driver_override)
276 found_id = &pci_device_id_any;
277
278 return found_id;
279}
280
281struct drv_dev_and_id {
282 struct pci_driver *drv;
283 struct pci_dev *dev;
284 const struct pci_device_id *id;
285};
286
287static long local_pci_probe(void *_ddi)
288{
289 struct drv_dev_and_id *ddi = _ddi;
290 struct pci_dev *pci_dev = ddi->dev;
291 struct pci_driver *pci_drv = ddi->drv;
292 struct device *dev = &pci_dev->dev;
293 int rc;
294
295
296
297
298
299
300
301
302
303
304 pm_runtime_get_sync(dev);
305 pci_dev->driver = pci_drv;
306 rc = pci_drv->probe(pci_dev, ddi->id);
307 if (!rc)
308 return rc;
309 if (rc < 0) {
310 pci_dev->driver = NULL;
311 pm_runtime_put_sync(dev);
312 return rc;
313 }
314
315
316
317
318 dev_warn(dev, "Driver probe function unexpectedly returned %d\n", rc);
319 return 0;
320}
321
322static bool pci_physfn_is_probed(struct pci_dev *dev)
323{
324#ifdef CONFIG_PCI_IOV
325 return dev->is_virtfn && dev->physfn->is_probed;
326#else
327 return false;
328#endif
329}
330
331static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev,
332 const struct pci_device_id *id)
333{
334 int error, node, cpu;
335 struct drv_dev_and_id ddi = { drv, dev, id };
336
337
338
339
340
341
342 node = dev_to_node(&dev->dev);
343 dev->is_probed = 1;
344
345 cpu_hotplug_disable();
346
347
348
349
350
351 if (node < 0 || node >= MAX_NUMNODES || !node_online(node) ||
352 pci_physfn_is_probed(dev))
353 cpu = nr_cpu_ids;
354 else
355 cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
356
357 if (cpu < nr_cpu_ids)
358 error = work_on_cpu(cpu, local_pci_probe, &ddi);
359 else
360 error = local_pci_probe(&ddi);
361
362 dev->is_probed = 0;
363 cpu_hotplug_enable();
364 return error;
365}
366
367
368
369
370
371
372
373
374
375static int __pci_device_probe(struct pci_driver *drv, struct pci_dev *pci_dev)
376{
377 const struct pci_device_id *id;
378 int error = 0;
379
380 if (!pci_dev->driver && drv->probe) {
381 error = -ENODEV;
382
383 id = pci_match_device(drv, pci_dev);
384 if (id)
385 error = pci_call_probe(drv, pci_dev, id);
386 }
387 return error;
388}
389
390int __weak pcibios_alloc_irq(struct pci_dev *dev)
391{
392 return 0;
393}
394
395void __weak pcibios_free_irq(struct pci_dev *dev)
396{
397}
398
399#ifdef CONFIG_PCI_IOV
400static inline bool pci_device_can_probe(struct pci_dev *pdev)
401{
402 return (!pdev->is_virtfn || pdev->physfn->sriov->drivers_autoprobe);
403}
404#else
405static inline bool pci_device_can_probe(struct pci_dev *pdev)
406{
407 return true;
408}
409#endif
410
411static int pci_device_probe(struct device *dev)
412{
413 int error;
414 struct pci_dev *pci_dev = to_pci_dev(dev);
415 struct pci_driver *drv = to_pci_driver(dev->driver);
416
417 pci_assign_irq(pci_dev);
418
419 error = pcibios_alloc_irq(pci_dev);
420 if (error < 0)
421 return error;
422
423 pci_dev_get(pci_dev);
424 if (pci_device_can_probe(pci_dev)) {
425 error = __pci_device_probe(drv, pci_dev);
426 if (error) {
427 pcibios_free_irq(pci_dev);
428 pci_dev_put(pci_dev);
429 }
430 }
431
432 return error;
433}
434
435static int pci_device_remove(struct device *dev)
436{
437 struct pci_dev *pci_dev = to_pci_dev(dev);
438 struct pci_driver *drv = pci_dev->driver;
439
440 if (drv) {
441 if (drv->remove) {
442 pm_runtime_get_sync(dev);
443 drv->remove(pci_dev);
444 pm_runtime_put_noidle(dev);
445 }
446 pcibios_free_irq(pci_dev);
447 pci_dev->driver = NULL;
448 pci_iov_remove(pci_dev);
449 }
450
451
452 pm_runtime_put_sync(dev);
453
454
455
456
457
458 if (pci_dev->current_state == PCI_D0)
459 pci_dev->current_state = PCI_UNKNOWN;
460
461
462
463
464
465
466
467
468
469
470 pci_dev_put(pci_dev);
471 return 0;
472}
473
474static void pci_device_shutdown(struct device *dev)
475{
476 struct pci_dev *pci_dev = to_pci_dev(dev);
477 struct pci_driver *drv = pci_dev->driver;
478
479 pm_runtime_resume(dev);
480
481 if (drv && drv->shutdown)
482 drv->shutdown(pci_dev);
483
484
485
486
487
488
489
490
491 if (kexec_in_progress && (pci_dev->current_state <= PCI_D3hot))
492 pci_clear_master(pci_dev);
493}
494
495#ifdef CONFIG_PM
496
497
498
499
500
501
502
503static int pci_restore_standard_config(struct pci_dev *pci_dev)
504{
505 pci_update_current_state(pci_dev, PCI_UNKNOWN);
506
507 if (pci_dev->current_state != PCI_D0) {
508 int error = pci_set_power_state(pci_dev, PCI_D0);
509 if (error)
510 return error;
511 }
512
513 pci_restore_state(pci_dev);
514 pci_pme_restore(pci_dev);
515 return 0;
516}
517
518#endif
519
520#ifdef CONFIG_PM_SLEEP
521
522static void pci_pm_default_resume_early(struct pci_dev *pci_dev)
523{
524 pci_power_up(pci_dev);
525 pci_restore_state(pci_dev);
526 pci_pme_restore(pci_dev);
527 pci_fixup_device(pci_fixup_resume_early, pci_dev);
528}
529
530
531
532
533
534static void pci_pm_set_unknown_state(struct pci_dev *pci_dev)
535{
536
537
538
539
540 if (pci_dev->current_state == PCI_D0)
541 pci_dev->current_state = PCI_UNKNOWN;
542}
543
544
545
546
547
548static int pci_pm_reenable_device(struct pci_dev *pci_dev)
549{
550 int retval;
551
552
553 retval = pci_reenable_device(pci_dev);
554
555
556
557
558 if (pci_dev->is_busmaster)
559 pci_set_master(pci_dev);
560
561 return retval;
562}
563
564static int pci_legacy_suspend(struct device *dev, pm_message_t state)
565{
566 struct pci_dev *pci_dev = to_pci_dev(dev);
567 struct pci_driver *drv = pci_dev->driver;
568
569 if (drv && drv->suspend) {
570 pci_power_t prev = pci_dev->current_state;
571 int error;
572
573 error = drv->suspend(pci_dev, state);
574 suspend_report_result(drv->suspend, error);
575 if (error)
576 return error;
577
578 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
579 && pci_dev->current_state != PCI_UNKNOWN) {
580 WARN_ONCE(pci_dev->current_state != prev,
581 "PCI PM: Device state not saved by %pF\n",
582 drv->suspend);
583 }
584 }
585
586 pci_fixup_device(pci_fixup_suspend, pci_dev);
587
588 return 0;
589}
590
591static int pci_legacy_suspend_late(struct device *dev, pm_message_t state)
592{
593 struct pci_dev *pci_dev = to_pci_dev(dev);
594 struct pci_driver *drv = pci_dev->driver;
595
596 if (drv && drv->suspend_late) {
597 pci_power_t prev = pci_dev->current_state;
598 int error;
599
600 error = drv->suspend_late(pci_dev, state);
601 suspend_report_result(drv->suspend_late, error);
602 if (error)
603 return error;
604
605 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
606 && pci_dev->current_state != PCI_UNKNOWN) {
607 WARN_ONCE(pci_dev->current_state != prev,
608 "PCI PM: Device state not saved by %pF\n",
609 drv->suspend_late);
610 goto Fixup;
611 }
612 }
613
614 if (!pci_dev->state_saved)
615 pci_save_state(pci_dev);
616
617 pci_pm_set_unknown_state(pci_dev);
618
619Fixup:
620 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
621
622 return 0;
623}
624
625static int pci_legacy_resume_early(struct device *dev)
626{
627 struct pci_dev *pci_dev = to_pci_dev(dev);
628 struct pci_driver *drv = pci_dev->driver;
629
630 return drv && drv->resume_early ?
631 drv->resume_early(pci_dev) : 0;
632}
633
634static int pci_legacy_resume(struct device *dev)
635{
636 struct pci_dev *pci_dev = to_pci_dev(dev);
637 struct pci_driver *drv = pci_dev->driver;
638
639 pci_fixup_device(pci_fixup_resume, pci_dev);
640
641 return drv && drv->resume ?
642 drv->resume(pci_dev) : pci_pm_reenable_device(pci_dev);
643}
644
645
646
647static void pci_pm_default_resume(struct pci_dev *pci_dev)
648{
649 pci_fixup_device(pci_fixup_resume, pci_dev);
650 pci_enable_wake(pci_dev, PCI_D0, false);
651}
652
653static void pci_pm_default_suspend(struct pci_dev *pci_dev)
654{
655
656 if (!pci_has_subordinate(pci_dev))
657 pci_disable_enabled_device(pci_dev);
658}
659
660static bool pci_has_legacy_pm_support(struct pci_dev *pci_dev)
661{
662 struct pci_driver *drv = pci_dev->driver;
663 bool ret = drv && (drv->suspend || drv->suspend_late || drv->resume
664 || drv->resume_early);
665
666
667
668
669
670
671 WARN(ret && drv->driver.pm, "driver %s device %04x:%04x\n",
672 drv->name, pci_dev->vendor, pci_dev->device);
673
674 return ret;
675}
676
677
678
679static int pci_pm_prepare(struct device *dev)
680{
681 struct device_driver *drv = dev->driver;
682
683 if (drv && drv->pm && drv->pm->prepare) {
684 int error = drv->pm->prepare(dev);
685 if (error < 0)
686 return error;
687
688 if (!error && dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_PREPARE))
689 return 0;
690 }
691 return pci_dev_keep_suspended(to_pci_dev(dev));
692}
693
694static void pci_pm_complete(struct device *dev)
695{
696 struct pci_dev *pci_dev = to_pci_dev(dev);
697
698 pci_dev_complete_resume(pci_dev);
699 pm_generic_complete(dev);
700
701
702 if (pm_runtime_suspended(dev) && pm_resume_via_firmware()) {
703 pci_power_t pre_sleep_state = pci_dev->current_state;
704
705 pci_update_current_state(pci_dev, pci_dev->current_state);
706 if (pci_dev->current_state < pre_sleep_state)
707 pm_request_resume(dev);
708 }
709}
710
711#else
712
713#define pci_pm_prepare NULL
714#define pci_pm_complete NULL
715
716#endif
717
718#ifdef CONFIG_SUSPEND
719static void pcie_pme_root_status_cleanup(struct pci_dev *pci_dev)
720{
721
722
723
724
725
726 if (pci_is_pcie(pci_dev) &&
727 (pci_pcie_type(pci_dev) == PCI_EXP_TYPE_ROOT_PORT ||
728 pci_pcie_type(pci_dev) == PCI_EXP_TYPE_RC_EC))
729 pcie_clear_root_pme_status(pci_dev);
730}
731
732static int pci_pm_suspend(struct device *dev)
733{
734 struct pci_dev *pci_dev = to_pci_dev(dev);
735 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
736
737 if (pci_has_legacy_pm_support(pci_dev))
738 return pci_legacy_suspend(dev, PMSG_SUSPEND);
739
740 if (!pm) {
741 pci_pm_default_suspend(pci_dev);
742 return 0;
743 }
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
759 !pci_dev_keep_suspended(pci_dev)) {
760 pm_runtime_resume(dev);
761 pci_dev->state_saved = false;
762 }
763
764 if (pm->suspend) {
765 pci_power_t prev = pci_dev->current_state;
766 int error;
767
768 error = pm->suspend(dev);
769 suspend_report_result(pm->suspend, error);
770 if (error)
771 return error;
772
773 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
774 && pci_dev->current_state != PCI_UNKNOWN) {
775 WARN_ONCE(pci_dev->current_state != prev,
776 "PCI PM: State of device not saved by %pF\n",
777 pm->suspend);
778 }
779 }
780
781 return 0;
782}
783
784static int pci_pm_suspend_late(struct device *dev)
785{
786 if (dev_pm_smart_suspend_and_suspended(dev))
787 return 0;
788
789 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
790
791 return pm_generic_suspend_late(dev);
792}
793
794static int pci_pm_suspend_noirq(struct device *dev)
795{
796 struct pci_dev *pci_dev = to_pci_dev(dev);
797 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
798
799 if (dev_pm_smart_suspend_and_suspended(dev)) {
800 dev->power.may_skip_resume = true;
801 return 0;
802 }
803
804 if (pci_has_legacy_pm_support(pci_dev))
805 return pci_legacy_suspend_late(dev, PMSG_SUSPEND);
806
807 if (!pm) {
808 pci_save_state(pci_dev);
809 goto Fixup;
810 }
811
812 if (pm->suspend_noirq) {
813 pci_power_t prev = pci_dev->current_state;
814 int error;
815
816 error = pm->suspend_noirq(dev);
817 suspend_report_result(pm->suspend_noirq, error);
818 if (error)
819 return error;
820
821 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
822 && pci_dev->current_state != PCI_UNKNOWN) {
823 WARN_ONCE(pci_dev->current_state != prev,
824 "PCI PM: State of device not saved by %pF\n",
825 pm->suspend_noirq);
826 goto Fixup;
827 }
828 }
829
830 if (!pci_dev->state_saved) {
831 pci_save_state(pci_dev);
832 if (pci_power_manageable(pci_dev))
833 pci_prepare_to_sleep(pci_dev);
834 }
835
836 dev_dbg(dev, "PCI PM: Suspend power state: %s\n",
837 pci_power_name(pci_dev->current_state));
838
839 pci_pm_set_unknown_state(pci_dev);
840
841
842
843
844
845
846
847
848
849
850 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
851 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
852
853Fixup:
854 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
855
856
857
858
859
860
861
862
863 dev->power.may_skip_resume = device_may_wakeup(dev) ||
864 !device_can_wakeup(dev);
865
866 return 0;
867}
868
869static int pci_pm_resume_noirq(struct device *dev)
870{
871 struct pci_dev *pci_dev = to_pci_dev(dev);
872 struct device_driver *drv = dev->driver;
873 int error = 0;
874
875 if (dev_pm_may_skip_resume(dev))
876 return 0;
877
878
879
880
881
882
883 if (dev_pm_smart_suspend_and_suspended(dev))
884 pm_runtime_set_active(dev);
885
886 pci_pm_default_resume_early(pci_dev);
887
888 if (pci_has_legacy_pm_support(pci_dev))
889 return pci_legacy_resume_early(dev);
890
891 pcie_pme_root_status_cleanup(pci_dev);
892
893 if (drv && drv->pm && drv->pm->resume_noirq)
894 error = drv->pm->resume_noirq(dev);
895
896 return error;
897}
898
899static int pci_pm_resume(struct device *dev)
900{
901 struct pci_dev *pci_dev = to_pci_dev(dev);
902 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
903 int error = 0;
904
905
906
907
908
909 if (pci_dev->state_saved)
910 pci_restore_standard_config(pci_dev);
911
912 if (pci_has_legacy_pm_support(pci_dev))
913 return pci_legacy_resume(dev);
914
915 pci_pm_default_resume(pci_dev);
916
917 if (pm) {
918 if (pm->resume)
919 error = pm->resume(dev);
920 } else {
921 pci_pm_reenable_device(pci_dev);
922 }
923
924 return error;
925}
926
927#else
928
929#define pci_pm_suspend NULL
930#define pci_pm_suspend_late NULL
931#define pci_pm_suspend_noirq NULL
932#define pci_pm_resume NULL
933#define pci_pm_resume_noirq NULL
934
935#endif
936
937#ifdef CONFIG_HIBERNATE_CALLBACKS
938
939
940
941
942
943
944struct dev_pm_ops __weak pcibios_pm_ops;
945
946static int pci_pm_freeze(struct device *dev)
947{
948 struct pci_dev *pci_dev = to_pci_dev(dev);
949 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
950
951 if (pci_has_legacy_pm_support(pci_dev))
952 return pci_legacy_suspend(dev, PMSG_FREEZE);
953
954 if (!pm) {
955 pci_pm_default_suspend(pci_dev);
956 return 0;
957 }
958
959
960
961
962
963
964
965 if (!dev_pm_smart_suspend_and_suspended(dev)) {
966 pm_runtime_resume(dev);
967 pci_dev->state_saved = false;
968 }
969
970 if (pm->freeze) {
971 int error;
972
973 error = pm->freeze(dev);
974 suspend_report_result(pm->freeze, error);
975 if (error)
976 return error;
977 }
978
979 return 0;
980}
981
982static int pci_pm_freeze_late(struct device *dev)
983{
984 if (dev_pm_smart_suspend_and_suspended(dev))
985 return 0;
986
987 return pm_generic_freeze_late(dev);
988}
989
990static int pci_pm_freeze_noirq(struct device *dev)
991{
992 struct pci_dev *pci_dev = to_pci_dev(dev);
993 struct device_driver *drv = dev->driver;
994
995 if (dev_pm_smart_suspend_and_suspended(dev))
996 return 0;
997
998 if (pci_has_legacy_pm_support(pci_dev))
999 return pci_legacy_suspend_late(dev, PMSG_FREEZE);
1000
1001 if (drv && drv->pm && drv->pm->freeze_noirq) {
1002 int error;
1003
1004 error = drv->pm->freeze_noirq(dev);
1005 suspend_report_result(drv->pm->freeze_noirq, error);
1006 if (error)
1007 return error;
1008 }
1009
1010 if (!pci_dev->state_saved)
1011 pci_save_state(pci_dev);
1012
1013 pci_pm_set_unknown_state(pci_dev);
1014
1015 if (pcibios_pm_ops.freeze_noirq)
1016 return pcibios_pm_ops.freeze_noirq(dev);
1017
1018 return 0;
1019}
1020
1021static int pci_pm_thaw_noirq(struct device *dev)
1022{
1023 struct pci_dev *pci_dev = to_pci_dev(dev);
1024 struct device_driver *drv = dev->driver;
1025 int error = 0;
1026
1027
1028
1029
1030
1031
1032 if (dev_pm_smart_suspend_and_suspended(dev)) {
1033 dev_pm_skip_next_resume_phases(dev);
1034 return 0;
1035 }
1036
1037 if (pcibios_pm_ops.thaw_noirq) {
1038 error = pcibios_pm_ops.thaw_noirq(dev);
1039 if (error)
1040 return error;
1041 }
1042
1043 if (pci_has_legacy_pm_support(pci_dev))
1044 return pci_legacy_resume_early(dev);
1045
1046
1047
1048
1049
1050
1051 pci_set_power_state(pci_dev, PCI_D0);
1052 pci_restore_state(pci_dev);
1053
1054 if (drv && drv->pm && drv->pm->thaw_noirq)
1055 error = drv->pm->thaw_noirq(dev);
1056
1057 return error;
1058}
1059
1060static int pci_pm_thaw(struct device *dev)
1061{
1062 struct pci_dev *pci_dev = to_pci_dev(dev);
1063 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1064 int error = 0;
1065
1066 if (pci_has_legacy_pm_support(pci_dev))
1067 return pci_legacy_resume(dev);
1068
1069 if (pm) {
1070 if (pm->thaw)
1071 error = pm->thaw(dev);
1072 } else {
1073 pci_pm_reenable_device(pci_dev);
1074 }
1075
1076 pci_dev->state_saved = false;
1077
1078 return error;
1079}
1080
1081static int pci_pm_poweroff(struct device *dev)
1082{
1083 struct pci_dev *pci_dev = to_pci_dev(dev);
1084 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1085
1086 if (pci_has_legacy_pm_support(pci_dev))
1087 return pci_legacy_suspend(dev, PMSG_HIBERNATE);
1088
1089 if (!pm) {
1090 pci_pm_default_suspend(pci_dev);
1091 return 0;
1092 }
1093
1094
1095 if (!dev_pm_test_driver_flags(dev, DPM_FLAG_SMART_SUSPEND) ||
1096 !pci_dev_keep_suspended(pci_dev))
1097 pm_runtime_resume(dev);
1098
1099 pci_dev->state_saved = false;
1100 if (pm->poweroff) {
1101 int error;
1102
1103 error = pm->poweroff(dev);
1104 suspend_report_result(pm->poweroff, error);
1105 if (error)
1106 return error;
1107 }
1108
1109 return 0;
1110}
1111
1112static int pci_pm_poweroff_late(struct device *dev)
1113{
1114 if (dev_pm_smart_suspend_and_suspended(dev))
1115 return 0;
1116
1117 pci_fixup_device(pci_fixup_suspend, to_pci_dev(dev));
1118
1119 return pm_generic_poweroff_late(dev);
1120}
1121
1122static int pci_pm_poweroff_noirq(struct device *dev)
1123{
1124 struct pci_dev *pci_dev = to_pci_dev(dev);
1125 struct device_driver *drv = dev->driver;
1126
1127 if (dev_pm_smart_suspend_and_suspended(dev))
1128 return 0;
1129
1130 if (pci_has_legacy_pm_support(to_pci_dev(dev)))
1131 return pci_legacy_suspend_late(dev, PMSG_HIBERNATE);
1132
1133 if (!drv || !drv->pm) {
1134 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1135 return 0;
1136 }
1137
1138 if (drv->pm->poweroff_noirq) {
1139 int error;
1140
1141 error = drv->pm->poweroff_noirq(dev);
1142 suspend_report_result(drv->pm->poweroff_noirq, error);
1143 if (error)
1144 return error;
1145 }
1146
1147 if (!pci_dev->state_saved && !pci_has_subordinate(pci_dev))
1148 pci_prepare_to_sleep(pci_dev);
1149
1150
1151
1152
1153
1154 if (pci_dev->class == PCI_CLASS_SERIAL_USB_EHCI)
1155 pci_write_config_word(pci_dev, PCI_COMMAND, 0);
1156
1157 pci_fixup_device(pci_fixup_suspend_late, pci_dev);
1158
1159 if (pcibios_pm_ops.poweroff_noirq)
1160 return pcibios_pm_ops.poweroff_noirq(dev);
1161
1162 return 0;
1163}
1164
1165static int pci_pm_restore_noirq(struct device *dev)
1166{
1167 struct pci_dev *pci_dev = to_pci_dev(dev);
1168 struct device_driver *drv = dev->driver;
1169 int error = 0;
1170
1171
1172 if (dev_pm_smart_suspend_and_suspended(dev))
1173 pm_runtime_set_active(dev);
1174
1175 if (pcibios_pm_ops.restore_noirq) {
1176 error = pcibios_pm_ops.restore_noirq(dev);
1177 if (error)
1178 return error;
1179 }
1180
1181 pci_pm_default_resume_early(pci_dev);
1182
1183 if (pci_has_legacy_pm_support(pci_dev))
1184 return pci_legacy_resume_early(dev);
1185
1186 if (drv && drv->pm && drv->pm->restore_noirq)
1187 error = drv->pm->restore_noirq(dev);
1188
1189 return error;
1190}
1191
1192static int pci_pm_restore(struct device *dev)
1193{
1194 struct pci_dev *pci_dev = to_pci_dev(dev);
1195 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1196 int error = 0;
1197
1198
1199
1200
1201
1202 if (pci_dev->state_saved)
1203 pci_restore_standard_config(pci_dev);
1204
1205 if (pci_has_legacy_pm_support(pci_dev))
1206 return pci_legacy_resume(dev);
1207
1208 pci_pm_default_resume(pci_dev);
1209
1210 if (pm) {
1211 if (pm->restore)
1212 error = pm->restore(dev);
1213 } else {
1214 pci_pm_reenable_device(pci_dev);
1215 }
1216
1217 return error;
1218}
1219
1220#else
1221
1222#define pci_pm_freeze NULL
1223#define pci_pm_freeze_late NULL
1224#define pci_pm_freeze_noirq NULL
1225#define pci_pm_thaw NULL
1226#define pci_pm_thaw_noirq NULL
1227#define pci_pm_poweroff NULL
1228#define pci_pm_poweroff_late NULL
1229#define pci_pm_poweroff_noirq NULL
1230#define pci_pm_restore NULL
1231#define pci_pm_restore_noirq NULL
1232
1233#endif
1234
1235#ifdef CONFIG_PM
1236
1237static int pci_pm_runtime_suspend(struct device *dev)
1238{
1239 struct pci_dev *pci_dev = to_pci_dev(dev);
1240 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1241 pci_power_t prev = pci_dev->current_state;
1242 int error;
1243
1244
1245
1246
1247
1248
1249 if (!pci_dev->driver) {
1250 pci_save_state(pci_dev);
1251 return 0;
1252 }
1253
1254 if (!pm || !pm->runtime_suspend)
1255 return -ENOSYS;
1256
1257 pci_dev->state_saved = false;
1258 error = pm->runtime_suspend(dev);
1259 if (error) {
1260
1261
1262
1263
1264
1265 if (error == -EBUSY || error == -EAGAIN)
1266 dev_dbg(dev, "can't suspend now (%pf returned %d)\n",
1267 pm->runtime_suspend, error);
1268 else
1269 dev_err(dev, "can't suspend (%pf returned %d)\n",
1270 pm->runtime_suspend, error);
1271
1272 return error;
1273 }
1274
1275 pci_fixup_device(pci_fixup_suspend, pci_dev);
1276
1277 if (!pci_dev->state_saved && pci_dev->current_state != PCI_D0
1278 && pci_dev->current_state != PCI_UNKNOWN) {
1279 WARN_ONCE(pci_dev->current_state != prev,
1280 "PCI PM: State of device not saved by %pF\n",
1281 pm->runtime_suspend);
1282 return 0;
1283 }
1284
1285 if (!pci_dev->state_saved) {
1286 pci_save_state(pci_dev);
1287 pci_finish_runtime_suspend(pci_dev);
1288 }
1289
1290 return 0;
1291}
1292
1293static int pci_pm_runtime_resume(struct device *dev)
1294{
1295 int rc;
1296 struct pci_dev *pci_dev = to_pci_dev(dev);
1297 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1298
1299
1300
1301
1302
1303
1304 pci_restore_standard_config(pci_dev);
1305
1306 if (!pci_dev->driver)
1307 return 0;
1308
1309 if (!pm || !pm->runtime_resume)
1310 return -ENOSYS;
1311
1312 pci_fixup_device(pci_fixup_resume_early, pci_dev);
1313 pci_enable_wake(pci_dev, PCI_D0, false);
1314 pci_fixup_device(pci_fixup_resume, pci_dev);
1315
1316 rc = pm->runtime_resume(dev);
1317
1318 pci_dev->runtime_d3cold = false;
1319
1320 return rc;
1321}
1322
1323static int pci_pm_runtime_idle(struct device *dev)
1324{
1325 struct pci_dev *pci_dev = to_pci_dev(dev);
1326 const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL;
1327 int ret = 0;
1328
1329
1330
1331
1332
1333 if (!pci_dev->driver)
1334 return 0;
1335
1336 if (!pm)
1337 return -ENOSYS;
1338
1339 if (pm->runtime_idle)
1340 ret = pm->runtime_idle(dev);
1341
1342 return ret;
1343}
1344
1345static const struct dev_pm_ops pci_dev_pm_ops = {
1346 .prepare = pci_pm_prepare,
1347 .complete = pci_pm_complete,
1348 .suspend = pci_pm_suspend,
1349 .suspend_late = pci_pm_suspend_late,
1350 .resume = pci_pm_resume,
1351 .freeze = pci_pm_freeze,
1352 .freeze_late = pci_pm_freeze_late,
1353 .thaw = pci_pm_thaw,
1354 .poweroff = pci_pm_poweroff,
1355 .poweroff_late = pci_pm_poweroff_late,
1356 .restore = pci_pm_restore,
1357 .suspend_noirq = pci_pm_suspend_noirq,
1358 .resume_noirq = pci_pm_resume_noirq,
1359 .freeze_noirq = pci_pm_freeze_noirq,
1360 .thaw_noirq = pci_pm_thaw_noirq,
1361 .poweroff_noirq = pci_pm_poweroff_noirq,
1362 .restore_noirq = pci_pm_restore_noirq,
1363 .runtime_suspend = pci_pm_runtime_suspend,
1364 .runtime_resume = pci_pm_runtime_resume,
1365 .runtime_idle = pci_pm_runtime_idle,
1366};
1367
1368#define PCI_PM_OPS_PTR (&pci_dev_pm_ops)
1369
1370#else
1371
1372#define pci_pm_runtime_suspend NULL
1373#define pci_pm_runtime_resume NULL
1374#define pci_pm_runtime_idle NULL
1375
1376#define PCI_PM_OPS_PTR NULL
1377
1378#endif
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391int __pci_register_driver(struct pci_driver *drv, struct module *owner,
1392 const char *mod_name)
1393{
1394
1395 drv->driver.name = drv->name;
1396 drv->driver.bus = &pci_bus_type;
1397 drv->driver.owner = owner;
1398 drv->driver.mod_name = mod_name;
1399 drv->driver.groups = drv->groups;
1400
1401 spin_lock_init(&drv->dynids.lock);
1402 INIT_LIST_HEAD(&drv->dynids.list);
1403
1404
1405 return driver_register(&drv->driver);
1406}
1407EXPORT_SYMBOL(__pci_register_driver);
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419void pci_unregister_driver(struct pci_driver *drv)
1420{
1421 driver_unregister(&drv->driver);
1422 pci_free_dynids(drv);
1423}
1424EXPORT_SYMBOL(pci_unregister_driver);
1425
1426static struct pci_driver pci_compat_driver = {
1427 .name = "compat"
1428};
1429
1430
1431
1432
1433
1434
1435
1436
1437struct pci_driver *pci_dev_driver(const struct pci_dev *dev)
1438{
1439 if (dev->driver)
1440 return dev->driver;
1441 else {
1442 int i;
1443 for (i = 0; i <= PCI_ROM_RESOURCE; i++)
1444 if (dev->resource[i].flags & IORESOURCE_BUSY)
1445 return &pci_compat_driver;
1446 }
1447 return NULL;
1448}
1449EXPORT_SYMBOL(pci_dev_driver);
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460static int pci_bus_match(struct device *dev, struct device_driver *drv)
1461{
1462 struct pci_dev *pci_dev = to_pci_dev(dev);
1463 struct pci_driver *pci_drv;
1464 const struct pci_device_id *found_id;
1465
1466 if (!pci_dev->match_driver)
1467 return 0;
1468
1469 pci_drv = to_pci_driver(drv);
1470 found_id = pci_match_device(pci_drv, pci_dev);
1471 if (found_id)
1472 return 1;
1473
1474 return 0;
1475}
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489struct pci_dev *pci_dev_get(struct pci_dev *dev)
1490{
1491 if (dev)
1492 get_device(&dev->dev);
1493 return dev;
1494}
1495EXPORT_SYMBOL(pci_dev_get);
1496
1497
1498
1499
1500
1501
1502
1503
1504void pci_dev_put(struct pci_dev *dev)
1505{
1506 if (dev)
1507 put_device(&dev->dev);
1508}
1509EXPORT_SYMBOL(pci_dev_put);
1510
1511static int pci_uevent(struct device *dev, struct kobj_uevent_env *env)
1512{
1513 struct pci_dev *pdev;
1514
1515 if (!dev)
1516 return -ENODEV;
1517
1518 pdev = to_pci_dev(dev);
1519
1520 if (add_uevent_var(env, "PCI_CLASS=%04X", pdev->class))
1521 return -ENOMEM;
1522
1523 if (add_uevent_var(env, "PCI_ID=%04X:%04X", pdev->vendor, pdev->device))
1524 return -ENOMEM;
1525
1526 if (add_uevent_var(env, "PCI_SUBSYS_ID=%04X:%04X", pdev->subsystem_vendor,
1527 pdev->subsystem_device))
1528 return -ENOMEM;
1529
1530 if (add_uevent_var(env, "PCI_SLOT_NAME=%s", pci_name(pdev)))
1531 return -ENOMEM;
1532
1533 if (add_uevent_var(env, "MODALIAS=pci:v%08Xd%08Xsv%08Xsd%08Xbc%02Xsc%02Xi%02X",
1534 pdev->vendor, pdev->device,
1535 pdev->subsystem_vendor, pdev->subsystem_device,
1536 (u8)(pdev->class >> 16), (u8)(pdev->class >> 8),
1537 (u8)(pdev->class)))
1538 return -ENOMEM;
1539
1540 return 0;
1541}
1542
1543#if defined(CONFIG_PCIEPORTBUS) || defined(CONFIG_EEH)
1544
1545
1546
1547
1548
1549void pci_uevent_ers(struct pci_dev *pdev, enum pci_ers_result err_type)
1550{
1551 int idx = 0;
1552 char *envp[3];
1553
1554 switch (err_type) {
1555 case PCI_ERS_RESULT_NONE:
1556 case PCI_ERS_RESULT_CAN_RECOVER:
1557 envp[idx++] = "ERROR_EVENT=BEGIN_RECOVERY";
1558 envp[idx++] = "DEVICE_ONLINE=0";
1559 break;
1560 case PCI_ERS_RESULT_RECOVERED:
1561 envp[idx++] = "ERROR_EVENT=SUCCESSFUL_RECOVERY";
1562 envp[idx++] = "DEVICE_ONLINE=1";
1563 break;
1564 case PCI_ERS_RESULT_DISCONNECT:
1565 envp[idx++] = "ERROR_EVENT=FAILED_RECOVERY";
1566 envp[idx++] = "DEVICE_ONLINE=0";
1567 break;
1568 default:
1569 break;
1570 }
1571
1572 if (idx > 0) {
1573 envp[idx++] = NULL;
1574 kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, envp);
1575 }
1576}
1577#endif
1578
1579static int pci_bus_num_vf(struct device *dev)
1580{
1581 return pci_num_vf(to_pci_dev(dev));
1582}
1583
1584
1585
1586
1587
1588
1589
1590
1591static int pci_dma_configure(struct device *dev)
1592{
1593 struct device *bridge;
1594 int ret = 0;
1595
1596 bridge = pci_get_host_bridge_device(to_pci_dev(dev));
1597
1598 if (IS_ENABLED(CONFIG_OF) && bridge->parent &&
1599 bridge->parent->of_node) {
1600 ret = of_dma_configure(dev, bridge->parent->of_node, true);
1601 } else if (has_acpi_companion(bridge)) {
1602 struct acpi_device *adev = to_acpi_device_node(bridge->fwnode);
1603 enum dev_dma_attr attr = acpi_get_dma_attr(adev);
1604
1605 if (attr != DEV_DMA_NOT_SUPPORTED)
1606 ret = acpi_dma_configure(dev, attr);
1607 }
1608
1609 pci_put_host_bridge_device(bridge);
1610 return ret;
1611}
1612
1613struct bus_type pci_bus_type = {
1614 .name = "pci",
1615 .match = pci_bus_match,
1616 .uevent = pci_uevent,
1617 .probe = pci_device_probe,
1618 .remove = pci_device_remove,
1619 .shutdown = pci_device_shutdown,
1620 .dev_groups = pci_dev_groups,
1621 .bus_groups = pci_bus_groups,
1622 .drv_groups = pci_drv_groups,
1623 .pm = PCI_PM_OPS_PTR,
1624 .num_vf = pci_bus_num_vf,
1625 .dma_configure = pci_dma_configure,
1626};
1627EXPORT_SYMBOL(pci_bus_type);
1628
1629#ifdef CONFIG_PCIEPORTBUS
1630static int pcie_port_bus_match(struct device *dev, struct device_driver *drv)
1631{
1632 struct pcie_device *pciedev;
1633 struct pcie_port_service_driver *driver;
1634
1635 if (drv->bus != &pcie_port_bus_type || dev->bus != &pcie_port_bus_type)
1636 return 0;
1637
1638 pciedev = to_pcie_device(dev);
1639 driver = to_service_driver(drv);
1640
1641 if (driver->service != pciedev->service)
1642 return 0;
1643
1644 if (driver->port_type != PCIE_ANY_PORT &&
1645 driver->port_type != pci_pcie_type(pciedev->port))
1646 return 0;
1647
1648 return 1;
1649}
1650
1651struct bus_type pcie_port_bus_type = {
1652 .name = "pci_express",
1653 .match = pcie_port_bus_match,
1654};
1655EXPORT_SYMBOL_GPL(pcie_port_bus_type);
1656#endif
1657
1658static int __init pci_driver_init(void)
1659{
1660 int ret;
1661
1662 ret = bus_register(&pci_bus_type);
1663 if (ret)
1664 return ret;
1665
1666#ifdef CONFIG_PCIEPORTBUS
1667 ret = bus_register(&pcie_port_bus_type);
1668 if (ret)
1669 return ret;
1670#endif
1671 dma_debug_add_bus(&pci_bus_type);
1672 return 0;
1673}
1674postcore_initcall(pci_driver_init);
1675