1
2#include <linux/pci.h>
3#include <linux/module.h>
4#include <linux/slab.h>
5#include <linux/ioport.h>
6#include <linux/wait.h>
7
8#include "pci.h"
9
10
11
12
13
14
15DEFINE_RAW_SPINLOCK(pci_lock);
16
17
18
19
20
21
22
23#define PCI_byte_BAD 0
24#define PCI_word_BAD (pos & 1)
25#define PCI_dword_BAD (pos & 3)
26
27#ifdef CONFIG_PCI_LOCKLESS_CONFIG
28# define pci_lock_config(f) do { (void)(f); } while (0)
29# define pci_unlock_config(f) do { (void)(f); } while (0)
30#else
31# define pci_lock_config(f) raw_spin_lock_irqsave(&pci_lock, f)
32# define pci_unlock_config(f) raw_spin_unlock_irqrestore(&pci_lock, f)
33#endif
34
35#define PCI_OP_READ(size, type, len) \
36int noinline pci_bus_read_config_##size \
37 (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \
38{ \
39 int res; \
40 unsigned long flags; \
41 u32 data = 0; \
42 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \
43 pci_lock_config(flags); \
44 res = bus->ops->read(bus, devfn, pos, len, &data); \
45 if (res) \
46 PCI_SET_ERROR_RESPONSE(value); \
47 else \
48 *value = (type)data; \
49 pci_unlock_config(flags); \
50 return res; \
51}
52
53#define PCI_OP_WRITE(size, type, len) \
54int noinline pci_bus_write_config_##size \
55 (struct pci_bus *bus, unsigned int devfn, int pos, type value) \
56{ \
57 int res; \
58 unsigned long flags; \
59 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \
60 pci_lock_config(flags); \
61 res = bus->ops->write(bus, devfn, pos, len, value); \
62 pci_unlock_config(flags); \
63 return res; \
64}
65
66PCI_OP_READ(byte, u8, 1)
67PCI_OP_READ(word, u16, 2)
68PCI_OP_READ(dword, u32, 4)
69PCI_OP_WRITE(byte, u8, 1)
70PCI_OP_WRITE(word, u16, 2)
71PCI_OP_WRITE(dword, u32, 4)
72
73EXPORT_SYMBOL(pci_bus_read_config_byte);
74EXPORT_SYMBOL(pci_bus_read_config_word);
75EXPORT_SYMBOL(pci_bus_read_config_dword);
76EXPORT_SYMBOL(pci_bus_write_config_byte);
77EXPORT_SYMBOL(pci_bus_write_config_word);
78EXPORT_SYMBOL(pci_bus_write_config_dword);
79
80int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn,
81 int where, int size, u32 *val)
82{
83 void __iomem *addr;
84
85 addr = bus->ops->map_bus(bus, devfn, where);
86 if (!addr)
87 return PCIBIOS_DEVICE_NOT_FOUND;
88
89 if (size == 1)
90 *val = readb(addr);
91 else if (size == 2)
92 *val = readw(addr);
93 else
94 *val = readl(addr);
95
96 return PCIBIOS_SUCCESSFUL;
97}
98EXPORT_SYMBOL_GPL(pci_generic_config_read);
99
100int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn,
101 int where, int size, u32 val)
102{
103 void __iomem *addr;
104
105 addr = bus->ops->map_bus(bus, devfn, where);
106 if (!addr)
107 return PCIBIOS_DEVICE_NOT_FOUND;
108
109 if (size == 1)
110 writeb(val, addr);
111 else if (size == 2)
112 writew(val, addr);
113 else
114 writel(val, addr);
115
116 return PCIBIOS_SUCCESSFUL;
117}
118EXPORT_SYMBOL_GPL(pci_generic_config_write);
119
120int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn,
121 int where, int size, u32 *val)
122{
123 void __iomem *addr;
124
125 addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
126 if (!addr)
127 return PCIBIOS_DEVICE_NOT_FOUND;
128
129 *val = readl(addr);
130
131 if (size <= 2)
132 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1);
133
134 return PCIBIOS_SUCCESSFUL;
135}
136EXPORT_SYMBOL_GPL(pci_generic_config_read32);
137
138int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn,
139 int where, int size, u32 val)
140{
141 void __iomem *addr;
142 u32 mask, tmp;
143
144 addr = bus->ops->map_bus(bus, devfn, where & ~0x3);
145 if (!addr)
146 return PCIBIOS_DEVICE_NOT_FOUND;
147
148 if (size == 4) {
149 writel(val, addr);
150 return PCIBIOS_SUCCESSFUL;
151 }
152
153
154
155
156
157
158
159
160
161
162 dev_warn_ratelimited(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n",
163 size, pci_domain_nr(bus), bus->number,
164 PCI_SLOT(devfn), PCI_FUNC(devfn), where);
165
166 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8));
167 tmp = readl(addr) & mask;
168 tmp |= val << ((where & 0x3) * 8);
169 writel(tmp, addr);
170
171 return PCIBIOS_SUCCESSFUL;
172}
173EXPORT_SYMBOL_GPL(pci_generic_config_write32);
174
175
176
177
178
179
180
181
182struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops)
183{
184 struct pci_ops *old_ops;
185 unsigned long flags;
186
187 raw_spin_lock_irqsave(&pci_lock, flags);
188 old_ops = bus->ops;
189 bus->ops = ops;
190 raw_spin_unlock_irqrestore(&pci_lock, flags);
191 return old_ops;
192}
193EXPORT_SYMBOL(pci_bus_set_ops);
194
195
196
197
198
199
200
201
202
203static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait);
204
205static noinline void pci_wait_cfg(struct pci_dev *dev)
206 __must_hold(&pci_lock)
207{
208 do {
209 raw_spin_unlock_irq(&pci_lock);
210 wait_event(pci_cfg_wait, !dev->block_cfg_access);
211 raw_spin_lock_irq(&pci_lock);
212 } while (dev->block_cfg_access);
213}
214
215
216#define PCI_USER_READ_CONFIG(size, type) \
217int pci_user_read_config_##size \
218 (struct pci_dev *dev, int pos, type *val) \
219{ \
220 int ret = PCIBIOS_SUCCESSFUL; \
221 u32 data = -1; \
222 if (PCI_##size##_BAD) \
223 return -EINVAL; \
224 raw_spin_lock_irq(&pci_lock); \
225 if (unlikely(dev->block_cfg_access)) \
226 pci_wait_cfg(dev); \
227 ret = dev->bus->ops->read(dev->bus, dev->devfn, \
228 pos, sizeof(type), &data); \
229 raw_spin_unlock_irq(&pci_lock); \
230 if (ret) \
231 PCI_SET_ERROR_RESPONSE(val); \
232 else \
233 *val = (type)data; \
234 return pcibios_err_to_errno(ret); \
235} \
236EXPORT_SYMBOL_GPL(pci_user_read_config_##size);
237
238
239#define PCI_USER_WRITE_CONFIG(size, type) \
240int pci_user_write_config_##size \
241 (struct pci_dev *dev, int pos, type val) \
242{ \
243 int ret = PCIBIOS_SUCCESSFUL; \
244 if (PCI_##size##_BAD) \
245 return -EINVAL; \
246 raw_spin_lock_irq(&pci_lock); \
247 if (unlikely(dev->block_cfg_access)) \
248 pci_wait_cfg(dev); \
249 ret = dev->bus->ops->write(dev->bus, dev->devfn, \
250 pos, sizeof(type), val); \
251 raw_spin_unlock_irq(&pci_lock); \
252 return pcibios_err_to_errno(ret); \
253} \
254EXPORT_SYMBOL_GPL(pci_user_write_config_##size);
255
256PCI_USER_READ_CONFIG(byte, u8)
257PCI_USER_READ_CONFIG(word, u16)
258PCI_USER_READ_CONFIG(dword, u32)
259PCI_USER_WRITE_CONFIG(byte, u8)
260PCI_USER_WRITE_CONFIG(word, u16)
261PCI_USER_WRITE_CONFIG(dword, u32)
262
263
264
265
266
267
268
269
270
271void pci_cfg_access_lock(struct pci_dev *dev)
272{
273 might_sleep();
274
275 raw_spin_lock_irq(&pci_lock);
276 if (dev->block_cfg_access)
277 pci_wait_cfg(dev);
278 dev->block_cfg_access = 1;
279 raw_spin_unlock_irq(&pci_lock);
280}
281EXPORT_SYMBOL_GPL(pci_cfg_access_lock);
282
283
284
285
286
287
288
289
290
291bool pci_cfg_access_trylock(struct pci_dev *dev)
292{
293 unsigned long flags;
294 bool locked = true;
295
296 raw_spin_lock_irqsave(&pci_lock, flags);
297 if (dev->block_cfg_access)
298 locked = false;
299 else
300 dev->block_cfg_access = 1;
301 raw_spin_unlock_irqrestore(&pci_lock, flags);
302
303 return locked;
304}
305EXPORT_SYMBOL_GPL(pci_cfg_access_trylock);
306
307
308
309
310
311
312
313void pci_cfg_access_unlock(struct pci_dev *dev)
314{
315 unsigned long flags;
316
317 raw_spin_lock_irqsave(&pci_lock, flags);
318
319
320
321
322
323 WARN_ON(!dev->block_cfg_access);
324
325 dev->block_cfg_access = 0;
326 raw_spin_unlock_irqrestore(&pci_lock, flags);
327
328 wake_up_all(&pci_cfg_wait);
329}
330EXPORT_SYMBOL_GPL(pci_cfg_access_unlock);
331
332static inline int pcie_cap_version(const struct pci_dev *dev)
333{
334 return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS;
335}
336
337bool pcie_cap_has_lnkctl(const struct pci_dev *dev)
338{
339 int type = pci_pcie_type(dev);
340
341 return type == PCI_EXP_TYPE_ENDPOINT ||
342 type == PCI_EXP_TYPE_LEG_END ||
343 type == PCI_EXP_TYPE_ROOT_PORT ||
344 type == PCI_EXP_TYPE_UPSTREAM ||
345 type == PCI_EXP_TYPE_DOWNSTREAM ||
346 type == PCI_EXP_TYPE_PCI_BRIDGE ||
347 type == PCI_EXP_TYPE_PCIE_BRIDGE;
348}
349
350static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev)
351{
352 return pcie_downstream_port(dev) &&
353 pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT;
354}
355
356bool pcie_cap_has_rtctl(const struct pci_dev *dev)
357{
358 int type = pci_pcie_type(dev);
359
360 return type == PCI_EXP_TYPE_ROOT_PORT ||
361 type == PCI_EXP_TYPE_RC_EC;
362}
363
364static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos)
365{
366 if (!pci_is_pcie(dev))
367 return false;
368
369 switch (pos) {
370 case PCI_EXP_FLAGS:
371 return true;
372 case PCI_EXP_DEVCAP:
373 case PCI_EXP_DEVCTL:
374 case PCI_EXP_DEVSTA:
375 return true;
376 case PCI_EXP_LNKCAP:
377 case PCI_EXP_LNKCTL:
378 case PCI_EXP_LNKSTA:
379 return pcie_cap_has_lnkctl(dev);
380 case PCI_EXP_SLTCAP:
381 case PCI_EXP_SLTCTL:
382 case PCI_EXP_SLTSTA:
383 return pcie_cap_has_sltctl(dev);
384 case PCI_EXP_RTCTL:
385 case PCI_EXP_RTCAP:
386 case PCI_EXP_RTSTA:
387 return pcie_cap_has_rtctl(dev);
388 case PCI_EXP_DEVCAP2:
389 case PCI_EXP_DEVCTL2:
390 case PCI_EXP_LNKCAP2:
391 case PCI_EXP_LNKCTL2:
392 case PCI_EXP_LNKSTA2:
393 return pcie_cap_version(dev) > 1;
394 default:
395 return false;
396 }
397}
398
399
400
401
402
403
404int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val)
405{
406 int ret;
407
408 *val = 0;
409 if (pos & 1)
410 return PCIBIOS_BAD_REGISTER_NUMBER;
411
412 if (pcie_capability_reg_implemented(dev, pos)) {
413 ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val);
414
415
416
417
418
419 if (ret)
420 *val = 0;
421 return ret;
422 }
423
424
425
426
427
428
429
430
431 if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
432 pos == PCI_EXP_SLTSTA)
433 *val = PCI_EXP_SLTSTA_PDS;
434
435 return 0;
436}
437EXPORT_SYMBOL(pcie_capability_read_word);
438
439int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val)
440{
441 int ret;
442
443 *val = 0;
444 if (pos & 3)
445 return PCIBIOS_BAD_REGISTER_NUMBER;
446
447 if (pcie_capability_reg_implemented(dev, pos)) {
448 ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val);
449
450
451
452
453
454 if (ret)
455 *val = 0;
456 return ret;
457 }
458
459 if (pci_is_pcie(dev) && pcie_downstream_port(dev) &&
460 pos == PCI_EXP_SLTSTA)
461 *val = PCI_EXP_SLTSTA_PDS;
462
463 return 0;
464}
465EXPORT_SYMBOL(pcie_capability_read_dword);
466
467int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val)
468{
469 if (pos & 1)
470 return PCIBIOS_BAD_REGISTER_NUMBER;
471
472 if (!pcie_capability_reg_implemented(dev, pos))
473 return 0;
474
475 return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val);
476}
477EXPORT_SYMBOL(pcie_capability_write_word);
478
479int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val)
480{
481 if (pos & 3)
482 return PCIBIOS_BAD_REGISTER_NUMBER;
483
484 if (!pcie_capability_reg_implemented(dev, pos))
485 return 0;
486
487 return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val);
488}
489EXPORT_SYMBOL(pcie_capability_write_dword);
490
491int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos,
492 u16 clear, u16 set)
493{
494 int ret;
495 u16 val;
496
497 ret = pcie_capability_read_word(dev, pos, &val);
498 if (!ret) {
499 val &= ~clear;
500 val |= set;
501 ret = pcie_capability_write_word(dev, pos, val);
502 }
503
504 return ret;
505}
506EXPORT_SYMBOL(pcie_capability_clear_and_set_word);
507
508int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos,
509 u32 clear, u32 set)
510{
511 int ret;
512 u32 val;
513
514 ret = pcie_capability_read_dword(dev, pos, &val);
515 if (!ret) {
516 val &= ~clear;
517 val |= set;
518 ret = pcie_capability_write_dword(dev, pos, val);
519 }
520
521 return ret;
522}
523EXPORT_SYMBOL(pcie_capability_clear_and_set_dword);
524
525int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val)
526{
527 if (pci_dev_is_disconnected(dev)) {
528 PCI_SET_ERROR_RESPONSE(val);
529 return PCIBIOS_DEVICE_NOT_FOUND;
530 }
531 return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val);
532}
533EXPORT_SYMBOL(pci_read_config_byte);
534
535int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val)
536{
537 if (pci_dev_is_disconnected(dev)) {
538 PCI_SET_ERROR_RESPONSE(val);
539 return PCIBIOS_DEVICE_NOT_FOUND;
540 }
541 return pci_bus_read_config_word(dev->bus, dev->devfn, where, val);
542}
543EXPORT_SYMBOL(pci_read_config_word);
544
545int pci_read_config_dword(const struct pci_dev *dev, int where,
546 u32 *val)
547{
548 if (pci_dev_is_disconnected(dev)) {
549 PCI_SET_ERROR_RESPONSE(val);
550 return PCIBIOS_DEVICE_NOT_FOUND;
551 }
552 return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val);
553}
554EXPORT_SYMBOL(pci_read_config_dword);
555
556int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val)
557{
558 if (pci_dev_is_disconnected(dev))
559 return PCIBIOS_DEVICE_NOT_FOUND;
560 return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val);
561}
562EXPORT_SYMBOL(pci_write_config_byte);
563
564int pci_write_config_word(const struct pci_dev *dev, int where, u16 val)
565{
566 if (pci_dev_is_disconnected(dev))
567 return PCIBIOS_DEVICE_NOT_FOUND;
568 return pci_bus_write_config_word(dev->bus, dev->devfn, where, val);
569}
570EXPORT_SYMBOL(pci_write_config_word);
571
572int pci_write_config_dword(const struct pci_dev *dev, int where,
573 u32 val)
574{
575 if (pci_dev_is_disconnected(dev))
576 return PCIBIOS_DEVICE_NOT_FOUND;
577 return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val);
578}
579EXPORT_SYMBOL(pci_write_config_dword);
580