1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <asm/unaligned.h>
19#include <linux/kernel.h>
20#include <linux/module.h>
21#include <linux/kref.h>
22#include <linux/io.h>
23#include <linux/delay.h>
24#include <linux/interrupt.h>
25#include <linux/sort.h>
26#include <linux/sched.h>
27#include <linux/types.h>
28#include <linux/pci.h>
29
30#include "nfp_cpp.h"
31
32#include "nfp6000/nfp6000.h"
33
34#include "nfp6000_pcie.h"
35
36#define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0)
37#define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \
38 (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
39#define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x) (((_x) & 0x3) << 30)
40#define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x) (((_x) >> 30) & 0x3)
41#define NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x) (((_x) & 0x3) << 28)
42#define NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x) (((_x) >> 28) & 0x3)
43#define NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x) (((_x) & 0xffffff) << 0)
44#define NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x) (((_x) >> 0) & 0xffffff)
45#define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \
46 (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
47#define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x) (((_x) & 0x7f) << 24)
48#define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x) (((_x) >> 24) & 0x7f)
49#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x) (((_x) & 0x3ff) << 14)
50#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x) (((_x) >> 14) & 0x3ff)
51#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x) (((_x) & 0x3fff) << 0)
52#define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x) (((_x) >> 0) & 0x3fff)
53#define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \
54 (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
55#define NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x) (((_x) & 0xf) << 28)
56#define NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x) (((_x) >> 28) & 0xf)
57#define NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x) (((_x) & 0x1f) << 23)
58#define NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x) (((_x) >> 23) & 0x1f)
59#define NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x) (((_x) & 0x1f) << 18)
60#define NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x) (((_x) >> 18) & 0x1f)
61#define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x) (((_x) & 0xff) << 10)
62#define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x) (((_x) >> 10) & 0xff)
63#define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x) (((_x) & 0x3ff) << 0)
64#define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff)
65
66#define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x) (((_x) & 0x1f) << 16)
67#define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f)
68#define NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x) (((_x) & 0xffff) << 0)
69#define NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x) (((_x) >> 0) & 0xffff)
70#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x) (((_x) & 0x3) << 27)
71#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x) (((_x) >> 27) & 0x3)
72#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT 0
73#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT 1
74#define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE 3
75#define NFP_PCIE_BAR_PCIE2CPP_MapType(_x) (((_x) & 0x7) << 29)
76#define NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x) (((_x) >> 29) & 0x7)
77#define NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED 0
78#define NFP_PCIE_BAR_PCIE2CPP_MapType_BULK 1
79#define NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET 2
80#define NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL 3
81#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0 4
82#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1 5
83#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2 6
84#define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3 7
85#define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x) (((_x) & 0xf) << 23)
86#define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf)
87#define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x) (((_x) & 0x3) << 21)
88#define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3)
89#define NFP_PCIE_EM 0x020000
90#define NFP_PCIE_SRAM 0x000000
91
92
93
94
95#define NFP_PCI_MIN_MAP_SIZE 0x080000
96
97#define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize)
98#define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize)
99#define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2))
100#define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4))
101#define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4))
102
103#define NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \
104 (0x400 + ((bar) * 8 + (slot)) * 4)
105
106#define NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \
107 (((bar) * 8 + (slot)) * 4)
108
109
110
111
112
113#define NFP_PCIE_EXPLICIT_BARS 2
114
115struct nfp6000_pcie;
116struct nfp6000_area_priv;
117
118
119
120
121
122
123
124
125
126
127
128
129
130struct nfp_bar {
131 struct nfp6000_pcie *nfp;
132 u32 barcfg;
133 u64 base;
134 u64 mask;
135 u32 bitsize;
136 int index;
137 atomic_t refcnt;
138
139 void __iomem *iomem;
140 struct resource *resource;
141};
142
143#define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8)
144
145struct nfp6000_pcie {
146 struct pci_dev *pdev;
147 struct device *dev;
148
149
150 spinlock_t bar_lock;
151 int bars;
152 struct nfp_bar bar[NFP_PCI_BAR_MAX];
153 wait_queue_head_t bar_waiters;
154
155
156 struct {
157 void __iomem *csr;
158 void __iomem *em;
159 void __iomem *expl[4];
160 } iomem;
161
162
163 struct {
164 struct mutex mutex;
165 u8 master_id;
166 u8 signal_ref;
167 void __iomem *data;
168 struct {
169 void __iomem *addr;
170 int bitsize;
171 int free[4];
172 } group[4];
173 } expl;
174};
175
176static u32 nfp_bar_maptype(struct nfp_bar *bar)
177{
178 return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
179}
180
181static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar)
182{
183 return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8;
184}
185
186static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar)
187{
188 return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2)
189 + nfp_bar_resource_len(bar) * (bar->index & 7);
190}
191
192#define TARGET_WIDTH_32 4
193#define TARGET_WIDTH_64 8
194
195static int
196compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar,
197 u32 *bar_config, u64 *bar_base,
198 int tgt, int act, int tok, u64 offset, size_t size, int width)
199{
200 int bitsize;
201 u32 newcfg;
202
203 if (tgt >= NFP_CPP_NUM_TARGETS)
204 return -EINVAL;
205
206 switch (width) {
207 case 8:
208 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
209 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT);
210 break;
211 case 4:
212 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
213 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT);
214 break;
215 case 0:
216 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
217 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE);
218 break;
219 default:
220 return -EINVAL;
221 }
222
223 if (act != NFP_CPP_ACTION_RW && act != 0) {
224
225 u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1);
226
227 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
228 NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED);
229 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
230 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act);
231 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
232
233 if ((offset & mask) != ((offset + size - 1) & mask))
234 return -EINVAL;
235 offset &= mask;
236
237 bitsize = 40 - 16;
238 } else {
239 u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1);
240
241
242 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
243 NFP_PCIE_BAR_PCIE2CPP_MapType_BULK);
244 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
245 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
246
247 if ((offset & mask) != ((offset + size - 1) & mask))
248 return -EINVAL;
249
250 offset &= mask;
251
252 bitsize = 40 - 21;
253 }
254
255 if (bar->bitsize < bitsize)
256 return -EINVAL;
257
258 newcfg |= offset >> bitsize;
259
260 if (bar_base)
261 *bar_base = offset;
262
263 if (bar_config)
264 *bar_config = newcfg;
265
266 return 0;
267}
268
269static int
270nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg)
271{
272 int base, slot;
273 int xbar;
274
275 base = bar->index >> 3;
276 slot = bar->index & 7;
277
278 if (nfp->iomem.csr) {
279 xbar = NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(base, slot);
280 writel(newcfg, nfp->iomem.csr + xbar);
281
282 readl(nfp->iomem.csr + xbar);
283 } else {
284 xbar = NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(base, slot);
285 pci_write_config_dword(nfp->pdev, xbar, newcfg);
286 }
287
288 bar->barcfg = newcfg;
289
290 return 0;
291}
292
293static int
294reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar,
295 int tgt, int act, int tok, u64 offset, size_t size, int width)
296{
297 u64 newbase;
298 u32 newcfg;
299 int err;
300
301 err = compute_bar(nfp, bar, &newcfg, &newbase,
302 tgt, act, tok, offset, size, width);
303 if (err)
304 return err;
305
306 bar->base = newbase;
307
308 return nfp6000_bar_write(nfp, bar, newcfg);
309}
310
311
312static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok,
313 u64 offset, size_t size, int width)
314{
315 int bartgt, baract, bartok;
316 int barwidth;
317 u32 maptype;
318
319 maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
320 bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg);
321 bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg);
322 baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg);
323
324 barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg);
325 switch (barwidth) {
326 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT:
327 barwidth = 4;
328 break;
329 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT:
330 barwidth = 8;
331 break;
332 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE:
333 barwidth = 0;
334 break;
335 default:
336 barwidth = -1;
337 break;
338 }
339
340 switch (maptype) {
341 case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET:
342 bartok = -1;
343
344 case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK:
345 baract = NFP_CPP_ACTION_RW;
346 if (act == 0)
347 act = NFP_CPP_ACTION_RW;
348
349 case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED:
350 break;
351 default:
352
353 return 0;
354 }
355
356
357 if (barwidth != width)
358 return 0;
359
360 if ((bartgt < 0 || bartgt == tgt) &&
361 (bartok < 0 || bartok == tok) &&
362 (baract == act) &&
363 bar->base <= offset &&
364 (bar->base + (1 << bar->bitsize)) >= (offset + size))
365 return 1;
366
367
368 return 0;
369}
370
371static int
372find_matching_bar(struct nfp6000_pcie *nfp,
373 u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
374{
375 int n;
376
377 for (n = 0; n < nfp->bars; n++) {
378 struct nfp_bar *bar = &nfp->bar[n];
379
380 if (matching_bar(bar, tgt, act, tok, offset, size, width))
381 return n;
382 }
383
384 return -1;
385}
386
387
388static int
389find_unused_bar_noblock(const struct nfp6000_pcie *nfp,
390 int tgt, int act, int tok,
391 u64 offset, size_t size, int width)
392{
393 int n, busy = 0;
394
395 for (n = 0; n < nfp->bars; n++) {
396 const struct nfp_bar *bar = &nfp->bar[n];
397 int err;
398
399 if (!bar->bitsize)
400 continue;
401
402
403 err = compute_bar(nfp, bar, NULL, NULL,
404 tgt, act, tok, offset, size, width);
405 if (err)
406 continue;
407
408 if (!atomic_read(&bar->refcnt))
409 return n;
410
411 busy++;
412 }
413
414 if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n",
415 tgt, act, tok, offset, size, width))
416 return -EINVAL;
417
418 return -EAGAIN;
419}
420
421static int
422find_unused_bar_and_lock(struct nfp6000_pcie *nfp,
423 int tgt, int act, int tok,
424 u64 offset, size_t size, int width)
425{
426 unsigned long flags;
427 int n;
428
429 spin_lock_irqsave(&nfp->bar_lock, flags);
430
431 n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width);
432 if (n < 0)
433 spin_unlock_irqrestore(&nfp->bar_lock, flags);
434 else
435 __release(&nfp->bar_lock);
436
437 return n;
438}
439
440static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
441{
442 atomic_inc(&bar->refcnt);
443}
444
445static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
446{
447 if (atomic_dec_and_test(&bar->refcnt))
448 wake_up_interruptible(&nfp->bar_waiters);
449}
450
451static int
452nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum,
453 u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
454{
455 return wait_event_interruptible(nfp->bar_waiters,
456 (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok,
457 offset, size, width))
458 != -EAGAIN);
459}
460
461static int
462nfp_alloc_bar(struct nfp6000_pcie *nfp,
463 u32 tgt, u32 act, u32 tok,
464 u64 offset, size_t size, int width, int nonblocking)
465{
466 unsigned long irqflags;
467 int barnum, retval;
468
469 if (size > (1 << 24))
470 return -EINVAL;
471
472 spin_lock_irqsave(&nfp->bar_lock, irqflags);
473 barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width);
474 if (barnum >= 0) {
475
476 nfp_bar_get(nfp, &nfp->bar[barnum]);
477 spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
478 return barnum;
479 }
480
481 barnum = find_unused_bar_noblock(nfp, tgt, act, tok,
482 offset, size, width);
483 if (barnum < 0) {
484 if (nonblocking)
485 goto err_nobar;
486
487
488
489
490
491 spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
492 retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok,
493 offset, size, width);
494 if (retval)
495 return retval;
496 __acquire(&nfp->bar_lock);
497 }
498
499 nfp_bar_get(nfp, &nfp->bar[barnum]);
500 retval = reconfigure_bar(nfp, &nfp->bar[barnum],
501 tgt, act, tok, offset, size, width);
502 if (retval < 0) {
503 nfp_bar_put(nfp, &nfp->bar[barnum]);
504 barnum = retval;
505 }
506
507err_nobar:
508 spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
509 return barnum;
510}
511
512static void disable_bars(struct nfp6000_pcie *nfp);
513
514static int bar_cmp(const void *aptr, const void *bptr)
515{
516 const struct nfp_bar *a = aptr, *b = bptr;
517
518 if (a->bitsize == b->bitsize)
519 return a->index - b->index;
520 else
521 return a->bitsize - b->bitsize;
522}
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540static int enable_bars(struct nfp6000_pcie *nfp, u16 interface)
541{
542 const u32 barcfg_msix_general =
543 NFP_PCIE_BAR_PCIE2CPP_MapType(
544 NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) |
545 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT;
546 const u32 barcfg_msix_xpb =
547 NFP_PCIE_BAR_PCIE2CPP_MapType(
548 NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) |
549 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT |
550 NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(
551 NFP_CPP_TARGET_ISLAND_XPB);
552 const u32 barcfg_explicit[4] = {
553 NFP_PCIE_BAR_PCIE2CPP_MapType(
554 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0),
555 NFP_PCIE_BAR_PCIE2CPP_MapType(
556 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1),
557 NFP_PCIE_BAR_PCIE2CPP_MapType(
558 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2),
559 NFP_PCIE_BAR_PCIE2CPP_MapType(
560 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3),
561 };
562 char status_msg[196] = {};
563 int i, err, bars_free;
564 struct nfp_bar *bar;
565 int expl_groups;
566 char *msg, *end;
567
568 msg = status_msg +
569 snprintf(status_msg, sizeof(status_msg) - 1, "RESERVED BARs: ");
570 end = status_msg + sizeof(status_msg) - 1;
571
572 bar = &nfp->bar[0];
573 for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) {
574 struct resource *res;
575
576 res = &nfp->pdev->resource[(i >> 3) * 2];
577
578
579 if (!(resource_type(res) & IORESOURCE_MEM)) {
580 bar--;
581 continue;
582 }
583
584 bar->resource = res;
585 bar->barcfg = 0;
586
587 bar->nfp = nfp;
588 bar->index = i;
589 bar->mask = nfp_bar_resource_len(bar) - 1;
590 bar->bitsize = fls(bar->mask);
591 bar->base = 0;
592 bar->iomem = NULL;
593 }
594
595 nfp->bars = bar - &nfp->bar[0];
596 if (nfp->bars < 8) {
597 dev_err(nfp->dev, "No usable BARs found!\n");
598 return -EINVAL;
599 }
600
601 bars_free = nfp->bars;
602
603
604
605 mutex_init(&nfp->expl.mutex);
606
607 nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4)
608 << 4;
609 nfp->expl.signal_ref = 0x10;
610
611
612 bar = &nfp->bar[0];
613 if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE)
614 bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar),
615 nfp_bar_resource_len(bar));
616 if (bar->iomem) {
617 int pf;
618
619 msg += snprintf(msg, end - msg, "0.0: General/MSI-X SRAM, ");
620 atomic_inc(&bar->refcnt);
621 bars_free--;
622
623 nfp6000_bar_write(nfp, bar, barcfg_msix_general);
624
625 nfp->expl.data = bar->iomem + NFP_PCIE_SRAM + 0x1000;
626
627 switch (nfp->pdev->device) {
628 case PCI_DEVICE_ID_NETRONOME_NFP3800:
629 pf = nfp->pdev->devfn & 7;
630 nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf);
631 break;
632 case PCI_DEVICE_ID_NETRONOME_NFP4000:
633 case PCI_DEVICE_ID_NETRONOME_NFP5000:
634 case PCI_DEVICE_ID_NETRONOME_NFP6000:
635 nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0);
636 break;
637 default:
638 dev_err(nfp->dev, "Unsupported device ID: %04hx!\n",
639 nfp->pdev->device);
640 err = -EINVAL;
641 goto err_unmap_bar0;
642 }
643 nfp->iomem.em = bar->iomem + NFP_PCIE_EM;
644 }
645
646 switch (nfp->pdev->device) {
647 case PCI_DEVICE_ID_NETRONOME_NFP3800:
648 expl_groups = 1;
649 break;
650 case PCI_DEVICE_ID_NETRONOME_NFP4000:
651 case PCI_DEVICE_ID_NETRONOME_NFP5000:
652 case PCI_DEVICE_ID_NETRONOME_NFP6000:
653 expl_groups = 4;
654 break;
655 default:
656 dev_err(nfp->dev, "Unsupported device ID: %04hx!\n",
657 nfp->pdev->device);
658 err = -EINVAL;
659 goto err_unmap_bar0;
660 }
661
662
663 bar = &nfp->bar[1];
664 msg += snprintf(msg, end - msg, "0.1: PCIe XPB/MSI-X PBA, ");
665 atomic_inc(&bar->refcnt);
666 bars_free--;
667
668 nfp6000_bar_write(nfp, bar, barcfg_msix_xpb);
669
670
671 for (i = 0; i < 4; i++) {
672 int j;
673
674 if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) {
675 nfp->expl.group[i].bitsize = 0;
676 continue;
677 }
678
679 bar = &nfp->bar[4 + i];
680 bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar),
681 nfp_bar_resource_len(bar));
682 if (bar->iomem) {
683 msg += snprintf(msg, end - msg,
684 "0.%d: Explicit%d, ", 4 + i, i);
685 atomic_inc(&bar->refcnt);
686 bars_free--;
687
688 nfp->expl.group[i].bitsize = bar->bitsize;
689 nfp->expl.group[i].addr = bar->iomem;
690 nfp6000_bar_write(nfp, bar, barcfg_explicit[i]);
691
692 for (j = 0; j < 4; j++)
693 nfp->expl.group[i].free[j] = true;
694 }
695 nfp->iomem.expl[i] = bar->iomem;
696 }
697
698
699 sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]),
700 bar_cmp, NULL);
701
702 dev_info(nfp->dev, "%sfree: %d/%d\n", status_msg, bars_free, nfp->bars);
703
704 return 0;
705
706err_unmap_bar0:
707 if (nfp->bar[0].iomem)
708 iounmap(nfp->bar[0].iomem);
709 return err;
710}
711
712static void disable_bars(struct nfp6000_pcie *nfp)
713{
714 struct nfp_bar *bar = &nfp->bar[0];
715 int n;
716
717 for (n = 0; n < nfp->bars; n++, bar++) {
718 if (bar->iomem) {
719 iounmap(bar->iomem);
720 bar->iomem = NULL;
721 }
722 }
723}
724
725
726
727
728
729struct nfp6000_area_priv {
730 atomic_t refcnt;
731
732 struct nfp_bar *bar;
733 u32 bar_offset;
734
735 u32 target;
736 u32 action;
737 u32 token;
738 u64 offset;
739 struct {
740 int read;
741 int write;
742 int bar;
743 } width;
744 size_t size;
745
746 void __iomem *iomem;
747 phys_addr_t phys;
748 struct resource resource;
749};
750
751static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest,
752 unsigned long long address, unsigned long size)
753{
754 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
755 u32 target = NFP_CPP_ID_TARGET_of(dest);
756 u32 action = NFP_CPP_ID_ACTION_of(dest);
757 u32 token = NFP_CPP_ID_TOKEN_of(dest);
758 int pp;
759
760 pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address);
761 if (pp < 0)
762 return pp;
763
764 priv->width.read = PUSH_WIDTH(pp);
765 priv->width.write = PULL_WIDTH(pp);
766 if (priv->width.read > 0 &&
767 priv->width.write > 0 &&
768 priv->width.read != priv->width.write) {
769 return -EINVAL;
770 }
771
772 if (priv->width.read > 0)
773 priv->width.bar = priv->width.read;
774 else
775 priv->width.bar = priv->width.write;
776
777 atomic_set(&priv->refcnt, 0);
778 priv->bar = NULL;
779
780 priv->target = target;
781 priv->action = action;
782 priv->token = token;
783 priv->offset = address;
784 priv->size = size;
785 memset(&priv->resource, 0, sizeof(priv->resource));
786
787 return 0;
788}
789
790static void nfp6000_area_cleanup(struct nfp_cpp_area *area)
791{
792}
793
794static void priv_area_get(struct nfp_cpp_area *area)
795{
796 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
797
798 atomic_inc(&priv->refcnt);
799}
800
801static int priv_area_put(struct nfp_cpp_area *area)
802{
803 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
804
805 if (WARN_ON(!atomic_read(&priv->refcnt)))
806 return 0;
807
808 return atomic_dec_and_test(&priv->refcnt);
809}
810
811static int nfp6000_area_acquire(struct nfp_cpp_area *area)
812{
813 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
814 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
815 int barnum, err;
816
817 if (priv->bar) {
818
819 priv_area_get(area);
820 return 0;
821 }
822
823 barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token,
824 priv->offset, priv->size, priv->width.bar, 1);
825
826 if (barnum < 0) {
827 err = barnum;
828 goto err_alloc_bar;
829 }
830 priv->bar = &nfp->bar[barnum];
831
832
833 if (nfp_bar_maptype(priv->bar) ==
834 NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) {
835 priv->bar_offset = priv->offset &
836 (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1);
837 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(
838 priv->bar, priv->target);
839 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(
840 priv->bar, priv->token);
841 } else {
842 priv->bar_offset = priv->offset & priv->bar->mask;
843 }
844
845
846
847
848
849
850 priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset;
851 priv->resource.name = nfp_cpp_area_name(area);
852 priv->resource.start = priv->phys;
853 priv->resource.end = priv->resource.start + priv->size - 1;
854 priv->resource.flags = IORESOURCE_MEM;
855
856
857 if (priv->bar->iomem)
858 priv->iomem = priv->bar->iomem + priv->bar_offset;
859 else
860
861 priv->iomem = ioremap_nocache(priv->phys, priv->size);
862
863 if (IS_ERR_OR_NULL(priv->iomem)) {
864 dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n",
865 (int)priv->size, priv->bar->index);
866 err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem);
867 priv->iomem = NULL;
868 goto err_iomem_remap;
869 }
870
871 priv_area_get(area);
872 return 0;
873
874err_iomem_remap:
875 nfp_bar_put(nfp, priv->bar);
876 priv->bar = NULL;
877err_alloc_bar:
878 return err;
879}
880
881static void nfp6000_area_release(struct nfp_cpp_area *area)
882{
883 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
884 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
885
886 if (!priv_area_put(area))
887 return;
888
889 if (!priv->bar->iomem)
890 iounmap(priv->iomem);
891
892 nfp_bar_put(nfp, priv->bar);
893
894 priv->bar = NULL;
895 priv->iomem = NULL;
896}
897
898static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area)
899{
900 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
901
902 return priv->phys;
903}
904
905static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area)
906{
907 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
908
909 return priv->iomem;
910}
911
912static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area)
913{
914
915
916
917
918 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
919
920 return priv->bar->resource;
921}
922
923static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
924 unsigned long offset, unsigned int length)
925{
926 u64 __maybe_unused *wrptr64 = kernel_vaddr;
927 const u64 __iomem __maybe_unused *rdptr64;
928 struct nfp6000_area_priv *priv;
929 u32 *wrptr32 = kernel_vaddr;
930 const u32 __iomem *rdptr32;
931 int n, width;
932
933 priv = nfp_cpp_area_priv(area);
934 rdptr64 = priv->iomem + offset;
935 rdptr32 = priv->iomem + offset;
936
937 if (offset + length > priv->size)
938 return -EFAULT;
939
940 width = priv->width.read;
941 if (width <= 0)
942 return -EINVAL;
943
944
945 if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) &&
946 priv->action == NFP_CPP_ACTION_RW &&
947 (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
948 width = TARGET_WIDTH_32;
949
950
951 if ((priv->offset + offset) & (width - 1))
952 return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area),
953 NFP_CPP_ID(priv->target,
954 priv->action,
955 priv->token),
956 priv->offset + offset,
957 kernel_vaddr, length, width);
958
959 if (WARN_ON(!priv->bar))
960 return -EFAULT;
961
962 switch (width) {
963 case TARGET_WIDTH_32:
964 if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
965 return -EINVAL;
966
967 for (n = 0; n < length; n += sizeof(u32))
968 *wrptr32++ = __raw_readl(rdptr32++);
969 return n;
970#ifdef __raw_readq
971 case TARGET_WIDTH_64:
972 if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
973 return -EINVAL;
974
975 for (n = 0; n < length; n += sizeof(u64))
976 *wrptr64++ = __raw_readq(rdptr64++);
977 return n;
978#endif
979 default:
980 return -EINVAL;
981 }
982}
983
984static int
985nfp6000_area_write(struct nfp_cpp_area *area,
986 const void *kernel_vaddr,
987 unsigned long offset, unsigned int length)
988{
989 const u64 __maybe_unused *rdptr64 = kernel_vaddr;
990 u64 __iomem __maybe_unused *wrptr64;
991 const u32 *rdptr32 = kernel_vaddr;
992 struct nfp6000_area_priv *priv;
993 u32 __iomem *wrptr32;
994 int n, width;
995
996 priv = nfp_cpp_area_priv(area);
997 wrptr64 = priv->iomem + offset;
998 wrptr32 = priv->iomem + offset;
999
1000 if (offset + length > priv->size)
1001 return -EFAULT;
1002
1003 width = priv->width.write;
1004 if (width <= 0)
1005 return -EINVAL;
1006
1007
1008 if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
1009 priv->action == NFP_CPP_ACTION_RW &&
1010 (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
1011 width = TARGET_WIDTH_32;
1012
1013
1014 if ((priv->offset + offset) & (width - 1))
1015 return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area),
1016 NFP_CPP_ID(priv->target,
1017 priv->action,
1018 priv->token),
1019 priv->offset + offset,
1020 kernel_vaddr, length, width);
1021
1022 if (WARN_ON(!priv->bar))
1023 return -EFAULT;
1024
1025 switch (width) {
1026 case TARGET_WIDTH_32:
1027 if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
1028 return -EINVAL;
1029
1030 for (n = 0; n < length; n += sizeof(u32)) {
1031 __raw_writel(*rdptr32++, wrptr32++);
1032 wmb();
1033 }
1034 return n;
1035#ifdef __raw_writeq
1036 case TARGET_WIDTH_64:
1037 if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
1038 return -EINVAL;
1039
1040 for (n = 0; n < length; n += sizeof(u64)) {
1041 __raw_writeq(*rdptr64++, wrptr64++);
1042 wmb();
1043 }
1044 return n;
1045#endif
1046 default:
1047 return -EINVAL;
1048 }
1049}
1050
1051struct nfp6000_explicit_priv {
1052 struct nfp6000_pcie *nfp;
1053 struct {
1054 int group;
1055 int area;
1056 } bar;
1057 int bitsize;
1058 void __iomem *data;
1059 void __iomem *addr;
1060};
1061
1062static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl)
1063{
1064 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl));
1065 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1066 int i, j;
1067
1068 mutex_lock(&nfp->expl.mutex);
1069 for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) {
1070 if (!nfp->expl.group[i].bitsize)
1071 continue;
1072
1073 for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) {
1074 u16 data_offset;
1075
1076 if (!nfp->expl.group[i].free[j])
1077 continue;
1078
1079 priv->nfp = nfp;
1080 priv->bar.group = i;
1081 priv->bar.area = j;
1082 priv->bitsize = nfp->expl.group[i].bitsize - 2;
1083
1084 data_offset = (priv->bar.group << 9) +
1085 (priv->bar.area << 7);
1086 priv->data = nfp->expl.data + data_offset;
1087 priv->addr = nfp->expl.group[i].addr +
1088 (priv->bar.area << priv->bitsize);
1089 nfp->expl.group[i].free[j] = false;
1090
1091 mutex_unlock(&nfp->expl.mutex);
1092 return 0;
1093 }
1094 }
1095 mutex_unlock(&nfp->expl.mutex);
1096
1097 return -EAGAIN;
1098}
1099
1100static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl)
1101{
1102 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1103 struct nfp6000_pcie *nfp = priv->nfp;
1104
1105 mutex_lock(&nfp->expl.mutex);
1106 nfp->expl.group[priv->bar.group].free[priv->bar.area] = true;
1107 mutex_unlock(&nfp->expl.mutex);
1108}
1109
1110static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl,
1111 const void *buff, size_t len)
1112{
1113 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1114 const u32 *src = buff;
1115 size_t i;
1116
1117 for (i = 0; i < len; i += sizeof(u32))
1118 writel(*(src++), priv->data + i);
1119
1120 return i;
1121}
1122
1123static int
1124nfp6000_explicit_do(struct nfp_cpp_explicit *expl,
1125 const struct nfp_cpp_explicit_command *cmd, u64 address)
1126{
1127 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1128 u8 signal_master, signal_ref, data_master;
1129 struct nfp6000_pcie *nfp = priv->nfp;
1130 int sigmask = 0;
1131 u16 data_ref;
1132 u32 csr[3];
1133
1134 if (cmd->siga_mode)
1135 sigmask |= 1 << cmd->siga;
1136 if (cmd->sigb_mode)
1137 sigmask |= 1 << cmd->sigb;
1138
1139 signal_master = cmd->signal_master;
1140 if (!signal_master)
1141 signal_master = nfp->expl.master_id;
1142
1143 signal_ref = cmd->signal_ref;
1144 if (signal_master == nfp->expl.master_id)
1145 signal_ref = nfp->expl.signal_ref +
1146 ((priv->bar.group * 4 + priv->bar.area) << 1);
1147
1148 data_master = cmd->data_master;
1149 if (!data_master)
1150 data_master = nfp->expl.master_id;
1151
1152 data_ref = cmd->data_ref;
1153 if (data_master == nfp->expl.master_id)
1154 data_ref = 0x1000 +
1155 (priv->bar.group << 9) + (priv->bar.area << 7);
1156
1157 csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) |
1158 NFP_PCIE_BAR_EXPLICIT_BAR0_Token(
1159 NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) |
1160 NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16);
1161
1162 csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) |
1163 NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) |
1164 NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref);
1165
1166 csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target(
1167 NFP_CPP_ID_TARGET_of(cmd->cpp_id)) |
1168 NFP_PCIE_BAR_EXPLICIT_BAR2_Action(
1169 NFP_CPP_ID_ACTION_of(cmd->cpp_id)) |
1170 NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) |
1171 NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) |
1172 NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master);
1173
1174 if (nfp->iomem.csr) {
1175 writel(csr[0], nfp->iomem.csr +
1176 NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1177 priv->bar.area));
1178 writel(csr[1], nfp->iomem.csr +
1179 NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1180 priv->bar.area));
1181 writel(csr[2], nfp->iomem.csr +
1182 NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1183 priv->bar.area));
1184
1185 readl(nfp->iomem.csr +
1186 NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1187 priv->bar.area));
1188 readl(nfp->iomem.csr +
1189 NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1190 priv->bar.area));
1191 readl(nfp->iomem.csr +
1192 NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1193 priv->bar.area));
1194 } else {
1195 pci_write_config_dword(nfp->pdev, 0x400 +
1196 NFP_PCIE_BAR_EXPLICIT_BAR0(
1197 priv->bar.group, priv->bar.area),
1198 csr[0]);
1199
1200 pci_write_config_dword(nfp->pdev, 0x400 +
1201 NFP_PCIE_BAR_EXPLICIT_BAR1(
1202 priv->bar.group, priv->bar.area),
1203 csr[1]);
1204
1205 pci_write_config_dword(nfp->pdev, 0x400 +
1206 NFP_PCIE_BAR_EXPLICIT_BAR2(
1207 priv->bar.group, priv->bar.area),
1208 csr[2]);
1209 }
1210
1211
1212 readb(priv->addr + (address & ((1 << priv->bitsize) - 1)));
1213
1214 return sigmask;
1215}
1216
1217static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl,
1218 void *buff, size_t len)
1219{
1220 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1221 u32 *dst = buff;
1222 size_t i;
1223
1224 for (i = 0; i < len; i += sizeof(u32))
1225 *(dst++) = readl(priv->data + i);
1226
1227 return i;
1228}
1229
1230static int nfp6000_init(struct nfp_cpp *cpp)
1231{
1232 nfp_cpp_area_cache_add(cpp, SZ_64K);
1233 nfp_cpp_area_cache_add(cpp, SZ_64K);
1234 nfp_cpp_area_cache_add(cpp, SZ_256K);
1235
1236 return 0;
1237}
1238
1239static void nfp6000_free(struct nfp_cpp *cpp)
1240{
1241 struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp);
1242
1243 disable_bars(nfp);
1244 kfree(nfp);
1245}
1246
1247static int nfp6000_read_serial(struct device *dev, u8 *serial)
1248{
1249 struct pci_dev *pdev = to_pci_dev(dev);
1250 int pos;
1251 u32 reg;
1252
1253 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
1254 if (!pos) {
1255 dev_err(dev, "can't find PCIe Serial Number Capability\n");
1256 return -EINVAL;
1257 }
1258
1259 pci_read_config_dword(pdev, pos + 4, ®);
1260 put_unaligned_be16(reg >> 16, serial + 4);
1261 pci_read_config_dword(pdev, pos + 8, ®);
1262 put_unaligned_be32(reg, serial);
1263
1264 return 0;
1265}
1266
1267static int nfp6000_get_interface(struct device *dev)
1268{
1269 struct pci_dev *pdev = to_pci_dev(dev);
1270 int pos;
1271 u32 reg;
1272
1273 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
1274 if (!pos) {
1275 dev_err(dev, "can't find PCIe Serial Number Capability\n");
1276 return -EINVAL;
1277 }
1278
1279 pci_read_config_dword(pdev, pos + 4, ®);
1280
1281 return reg & 0xffff;
1282}
1283
1284static const struct nfp_cpp_operations nfp6000_pcie_ops = {
1285 .owner = THIS_MODULE,
1286
1287 .init = nfp6000_init,
1288 .free = nfp6000_free,
1289
1290 .read_serial = nfp6000_read_serial,
1291 .get_interface = nfp6000_get_interface,
1292
1293 .area_priv_size = sizeof(struct nfp6000_area_priv),
1294 .area_init = nfp6000_area_init,
1295 .area_cleanup = nfp6000_area_cleanup,
1296 .area_acquire = nfp6000_area_acquire,
1297 .area_release = nfp6000_area_release,
1298 .area_phys = nfp6000_area_phys,
1299 .area_iomem = nfp6000_area_iomem,
1300 .area_resource = nfp6000_area_resource,
1301 .area_read = nfp6000_area_read,
1302 .area_write = nfp6000_area_write,
1303
1304 .explicit_priv_size = sizeof(struct nfp6000_explicit_priv),
1305 .explicit_acquire = nfp6000_explicit_acquire,
1306 .explicit_release = nfp6000_explicit_release,
1307 .explicit_put = nfp6000_explicit_put,
1308 .explicit_do = nfp6000_explicit_do,
1309 .explicit_get = nfp6000_explicit_get,
1310};
1311
1312
1313
1314
1315
1316
1317
1318struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev)
1319{
1320 struct nfp6000_pcie *nfp;
1321 u16 interface;
1322 int err;
1323
1324
1325 dev_info(&pdev->dev,
1326 "Netronome Flow Processor NFP4000/NFP5000/NFP6000 PCIe Card Probe\n");
1327 pcie_print_link_status(pdev);
1328
1329 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
1330 if (!nfp) {
1331 err = -ENOMEM;
1332 goto err_ret;
1333 }
1334
1335 nfp->dev = &pdev->dev;
1336 nfp->pdev = pdev;
1337 init_waitqueue_head(&nfp->bar_waiters);
1338 spin_lock_init(&nfp->bar_lock);
1339
1340 interface = nfp6000_get_interface(&pdev->dev);
1341
1342 if (NFP_CPP_INTERFACE_TYPE_of(interface) !=
1343 NFP_CPP_INTERFACE_TYPE_PCI) {
1344 dev_err(&pdev->dev,
1345 "Interface type %d is not the expected %d\n",
1346 NFP_CPP_INTERFACE_TYPE_of(interface),
1347 NFP_CPP_INTERFACE_TYPE_PCI);
1348 err = -ENODEV;
1349 goto err_free_nfp;
1350 }
1351
1352 if (NFP_CPP_INTERFACE_CHANNEL_of(interface) !=
1353 NFP_CPP_INTERFACE_CHANNEL_PEROPENER) {
1354 dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n",
1355 NFP_CPP_INTERFACE_CHANNEL_of(interface),
1356 NFP_CPP_INTERFACE_CHANNEL_PEROPENER);
1357 err = -ENODEV;
1358 goto err_free_nfp;
1359 }
1360
1361 err = enable_bars(nfp, interface);
1362 if (err)
1363 goto err_free_nfp;
1364
1365
1366 return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp);
1367
1368err_free_nfp:
1369 kfree(nfp);
1370err_ret:
1371 dev_err(&pdev->dev, "NFP6000 PCI setup failed\n");
1372 return ERR_PTR(err);
1373}
1374
