1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#include <linux/debugfs.h>
49#include <linux/delay.h>
50#include <linux/init.h>
51#include <linux/interrupt.h>
52#include <linux/module.h>
53#include <linux/pci.h>
54#include <linux/random.h>
55#include <linux/slab.h>
56#include "ntb_hw.h"
57#include "ntb_regs.h"
58
59#define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver"
60#define NTB_VER "1.0"
61
62MODULE_DESCRIPTION(NTB_NAME);
63MODULE_VERSION(NTB_VER);
64MODULE_LICENSE("Dual BSD/GPL");
65MODULE_AUTHOR("Intel Corporation");
66
67static bool xeon_errata_workaround = true;
68module_param(xeon_errata_workaround, bool, 0644);
69MODULE_PARM_DESC(xeon_errata_workaround, "Workaround for the Xeon Errata");
70
71enum {
72 NTB_CONN_TRANSPARENT = 0,
73 NTB_CONN_B2B,
74 NTB_CONN_RP,
75};
76
77enum {
78 NTB_DEV_USD = 0,
79 NTB_DEV_DSD,
80};
81
82enum {
83 SNB_HW = 0,
84 BWD_HW,
85};
86
87static struct dentry *debugfs_dir;
88
89#define BWD_LINK_RECOVERY_TIME 500
90
91
92#define MW_TO_BAR(mw) (mw * NTB_MAX_NUM_MW + 2)
93
94static DEFINE_PCI_DEVICE_TABLE(ntb_pci_tbl) = {
95 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
96 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
97 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
98 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
99 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
100 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
101 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
102 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
103 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
104 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
105 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
106 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
107 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
108 {0}
109};
110MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
111
112
113
114
115
116
117
118
119
120
121
122int ntb_register_event_callback(struct ntb_device *ndev,
123 void (*func)(void *handle, enum ntb_hw_event event))
124{
125 if (ndev->event_cb)
126 return -EINVAL;
127
128 ndev->event_cb = func;
129
130 return 0;
131}
132
133
134
135
136
137
138
139void ntb_unregister_event_callback(struct ntb_device *ndev)
140{
141 ndev->event_cb = NULL;
142}
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
158 void *data, void (*func)(void *data, int db_num))
159{
160 unsigned long mask;
161
162 if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
163 dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
164 return -EINVAL;
165 }
166
167 ndev->db_cb[idx].callback = func;
168 ndev->db_cb[idx].data = data;
169
170
171 mask = readw(ndev->reg_ofs.ldb_mask);
172 clear_bit(idx * ndev->bits_per_vector, &mask);
173 writew(mask, ndev->reg_ofs.ldb_mask);
174
175 return 0;
176}
177
178
179
180
181
182
183
184
185
186void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
187{
188 unsigned long mask;
189
190 if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
191 return;
192
193 mask = readw(ndev->reg_ofs.ldb_mask);
194 set_bit(idx * ndev->bits_per_vector, &mask);
195 writew(mask, ndev->reg_ofs.ldb_mask);
196
197 ndev->db_cb[idx].callback = NULL;
198}
199
200
201
202
203
204
205
206
207
208
209void *ntb_find_transport(struct pci_dev *pdev)
210{
211 struct ntb_device *ndev = pci_get_drvdata(pdev);
212 return ndev->ntb_transport;
213}
214
215
216
217
218
219
220
221
222
223
224struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
225{
226 struct ntb_device *ndev = pci_get_drvdata(pdev);
227
228 if (ndev->ntb_transport)
229 return NULL;
230
231 ndev->ntb_transport = transport;
232 return ndev;
233}
234
235
236
237
238
239
240
241
242void ntb_unregister_transport(struct ntb_device *ndev)
243{
244 int i;
245
246 if (!ndev->ntb_transport)
247 return;
248
249 for (i = 0; i < ndev->max_cbs; i++)
250 ntb_unregister_db_callback(ndev, i);
251
252 ntb_unregister_event_callback(ndev);
253 ndev->ntb_transport = NULL;
254}
255
256
257
258
259
260
261
262
263
264
265
266
267
268int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
269{
270 if (idx >= ndev->limits.max_spads)
271 return -EINVAL;
272
273 dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
274 val, idx);
275 writel(val, ndev->reg_ofs.spad_read + idx * 4);
276
277 return 0;
278}
279
280
281
282
283
284
285
286
287
288
289
290
291
292int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
293{
294 if (idx >= ndev->limits.max_spads)
295 return -EINVAL;
296
297 *val = readl(ndev->reg_ofs.spad_write + idx * 4);
298 dev_dbg(&ndev->pdev->dev,
299 "Reading %x from local scratch pad index %d\n", *val, idx);
300
301 return 0;
302}
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
318{
319 if (idx >= ndev->limits.max_spads)
320 return -EINVAL;
321
322 dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
323 val, idx);
324 writel(val, ndev->reg_ofs.spad_write + idx * 4);
325
326 return 0;
327}
328
329
330
331
332
333
334
335
336
337
338
339
340
341int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
342{
343 if (idx >= ndev->limits.max_spads)
344 return -EINVAL;
345
346 *val = readl(ndev->reg_ofs.spad_read + idx * 4);
347 dev_dbg(&ndev->pdev->dev,
348 "Reading %x from remote scratch pad index %d\n", *val, idx);
349
350 return 0;
351}
352
353
354
355
356
357
358
359
360
361
362resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
363{
364 if (mw >= ntb_max_mw(ndev))
365 return 0;
366
367 return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
368}
369
370
371
372
373
374
375
376
377
378
379
380void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
381{
382 if (mw >= ntb_max_mw(ndev))
383 return NULL;
384
385 return ndev->mw[mw].vbase;
386}
387
388
389
390
391
392
393
394
395
396
397u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
398{
399 if (mw >= ntb_max_mw(ndev))
400 return 0;
401
402 return ndev->mw[mw].bar_sz;
403}
404
405
406
407
408
409
410
411
412
413
414
415void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
416{
417 if (mw >= ntb_max_mw(ndev))
418 return;
419
420 dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
421 MW_TO_BAR(mw));
422
423 ndev->mw[mw].phys_addr = addr;
424
425 switch (MW_TO_BAR(mw)) {
426 case NTB_BAR_23:
427 writeq(addr, ndev->reg_ofs.bar2_xlat);
428 break;
429 case NTB_BAR_45:
430 writeq(addr, ndev->reg_ofs.bar4_xlat);
431 break;
432 }
433}
434
435
436
437
438
439
440
441
442
443
444
445void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
446{
447 dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
448
449 if (ndev->hw_type == BWD_HW)
450 writeq((u64) 1 << db, ndev->reg_ofs.rdb);
451 else
452 writew(((1 << ndev->bits_per_vector) - 1) <<
453 (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
454}
455
456static void bwd_recover_link(struct ntb_device *ndev)
457{
458 u32 status;
459
460
461 writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
462 writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
463 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
464 writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
465
466
467 msleep(100);
468
469
470 status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
471 dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
472 status &= PCI_ERR_COR_REP_ROLL;
473 writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
474
475
476 status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
477 dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
478 status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
479 writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
480
481
482 status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
483 dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
484 status |= BWD_DESKEWSTS_DBERR;
485 writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
486
487 status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
488 dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
489 status &= BWD_IBIST_ERR_OFLOW;
490 writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
491
492
493 status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
494 dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
495 status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
496 writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
497}
498
499static void ntb_link_event(struct ntb_device *ndev, int link_state)
500{
501 unsigned int event;
502
503 if (ndev->link_status == link_state)
504 return;
505
506 if (link_state == NTB_LINK_UP) {
507 u16 status;
508
509 dev_info(&ndev->pdev->dev, "Link Up\n");
510 ndev->link_status = NTB_LINK_UP;
511 event = NTB_EVENT_HW_LINK_UP;
512
513 if (ndev->hw_type == BWD_HW ||
514 ndev->conn_type == NTB_CONN_TRANSPARENT)
515 status = readw(ndev->reg_ofs.lnk_stat);
516 else {
517 int rc = pci_read_config_word(ndev->pdev,
518 SNB_LINK_STATUS_OFFSET,
519 &status);
520 if (rc)
521 return;
522 }
523
524 ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
525 ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
526 dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
527 ndev->link_width, ndev->link_speed);
528 } else {
529 dev_info(&ndev->pdev->dev, "Link Down\n");
530 ndev->link_status = NTB_LINK_DOWN;
531 event = NTB_EVENT_HW_LINK_DOWN;
532
533 }
534
535
536 if (ndev->event_cb)
537 ndev->event_cb(ndev->ntb_transport, event);
538}
539
540static int ntb_link_status(struct ntb_device *ndev)
541{
542 int link_state;
543
544 if (ndev->hw_type == BWD_HW) {
545 u32 ntb_cntl;
546
547 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
548 if (ntb_cntl & BWD_CNTL_LINK_DOWN)
549 link_state = NTB_LINK_DOWN;
550 else
551 link_state = NTB_LINK_UP;
552 } else {
553 u16 status;
554 int rc;
555
556 rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
557 &status);
558 if (rc)
559 return rc;
560
561 if (status & NTB_LINK_STATUS_ACTIVE)
562 link_state = NTB_LINK_UP;
563 else
564 link_state = NTB_LINK_DOWN;
565 }
566
567 ntb_link_event(ndev, link_state);
568
569 return 0;
570}
571
572static void bwd_link_recovery(struct work_struct *work)
573{
574 struct ntb_device *ndev = container_of(work, struct ntb_device,
575 lr_timer.work);
576 u32 status32;
577
578 bwd_recover_link(ndev);
579
580
581
582
583
584 msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
585
586 status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
587 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
588 goto retry;
589
590 status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
591 if (status32 & BWD_IBIST_ERR_OFLOW)
592 goto retry;
593
594 status32 = readl(ndev->reg_ofs.lnk_cntl);
595 if (!(status32 & BWD_CNTL_LINK_DOWN)) {
596 unsigned char speed, width;
597 u16 status16;
598
599 status16 = readw(ndev->reg_ofs.lnk_stat);
600 width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
601 speed = (status16 & NTB_LINK_SPEED_MASK);
602 if (ndev->link_width != width || ndev->link_speed != speed)
603 goto retry;
604 }
605
606 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
607 return;
608
609retry:
610 schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
611}
612
613
614static void bwd_link_poll(struct work_struct *work)
615{
616 struct ntb_device *ndev = container_of(work, struct ntb_device,
617 hb_timer.work);
618 unsigned long ts = jiffies;
619
620
621
622
623 if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
624 int rc = ntb_link_status(ndev);
625 if (rc)
626 dev_err(&ndev->pdev->dev,
627 "Error determining link status\n");
628
629
630 if (ndev->link_status == NTB_LINK_DOWN) {
631 u32 status32 = readl(ndev->reg_base +
632 BWD_LTSSMSTATEJMP_OFFSET);
633 if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
634 schedule_delayed_work(&ndev->lr_timer, 0);
635 return;
636 }
637 }
638 }
639
640 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
641}
642
643static int ntb_xeon_setup(struct ntb_device *ndev)
644{
645 int rc;
646 u8 val;
647
648 ndev->hw_type = SNB_HW;
649
650 rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &val);
651 if (rc)
652 return rc;
653
654 if (val & SNB_PPD_DEV_TYPE)
655 ndev->dev_type = NTB_DEV_USD;
656 else
657 ndev->dev_type = NTB_DEV_DSD;
658
659 switch (val & SNB_PPD_CONN_TYPE) {
660 case NTB_CONN_B2B:
661 dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
662 ndev->conn_type = NTB_CONN_B2B;
663 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
664 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
665 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
666 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
667 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
668 ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
669
670
671
672
673
674
675
676 if (xeon_errata_workaround) {
677 if (!ndev->mw[1].bar_sz)
678 return -EINVAL;
679
680 ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
681 ndev->reg_ofs.spad_write = ndev->mw[1].vbase +
682 SNB_SPAD_OFFSET;
683 ndev->reg_ofs.rdb = ndev->mw[1].vbase +
684 SNB_PDOORBELL_OFFSET;
685
686
687
688
689 writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
690 SNB_PBAR4LMT_OFFSET);
691 } else {
692 ndev->limits.max_mw = SNB_MAX_MW;
693 ndev->reg_ofs.spad_write = ndev->reg_base +
694 SNB_B2B_SPAD_OFFSET;
695 ndev->reg_ofs.rdb = ndev->reg_base +
696 SNB_B2B_DOORBELL_OFFSET;
697
698
699
700
701 writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
702 }
703
704
705
706
707
708 if (ndev->dev_type == NTB_DEV_USD) {
709 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
710 SNB_PBAR2XLAT_OFFSET);
711 if (xeon_errata_workaround)
712 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
713 SNB_PBAR4XLAT_OFFSET);
714 else {
715 writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
716 SNB_PBAR4XLAT_OFFSET);
717
718
719
720 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
721 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
722 writel(SNB_MBAR01_DSD_ADDR >> 32,
723 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
724 }
725
726 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
727 SNB_SBAR0BASE_OFFSET);
728 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
729 SNB_SBAR2BASE_OFFSET);
730 writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
731 SNB_SBAR4BASE_OFFSET);
732 } else {
733 writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
734 SNB_PBAR2XLAT_OFFSET);
735 if (xeon_errata_workaround)
736 writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
737 SNB_PBAR4XLAT_OFFSET);
738 else {
739 writeq(SNB_MBAR45_USD_ADDR, ndev->reg_base +
740 SNB_PBAR4XLAT_OFFSET);
741
742
743
744 writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
745 ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
746 writel(SNB_MBAR01_USD_ADDR >> 32,
747 ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
748 }
749 writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
750 SNB_SBAR0BASE_OFFSET);
751 writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
752 SNB_SBAR2BASE_OFFSET);
753 writeq(SNB_MBAR45_DSD_ADDR, ndev->reg_base +
754 SNB_SBAR4BASE_OFFSET);
755 }
756 break;
757 case NTB_CONN_RP:
758 dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
759 ndev->conn_type = NTB_CONN_RP;
760
761 if (xeon_errata_workaround) {
762 dev_err(&ndev->pdev->dev,
763 "NTB-RP disabled due to hardware errata. To disregard this warning and potentially lock-up the system, add the parameter 'xeon_errata_workaround=0'.\n");
764 return -EINVAL;
765 }
766
767
768
769
770
771 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
772
773
774
775 ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
776 ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
777 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
778
779
780
781 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
782 ndev->limits.max_spads * 4;
783 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
784 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
785 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
786 ndev->limits.max_mw = SNB_MAX_MW;
787 break;
788 case NTB_CONN_TRANSPARENT:
789 dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
790 ndev->conn_type = NTB_CONN_TRANSPARENT;
791
792
793
794
795 ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
796 ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
797 ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
798 ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
799 ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
800
801
802
803 ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
804 ndev->limits.max_spads * 4;
805 ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
806 ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
807
808 ndev->limits.max_mw = SNB_MAX_MW;
809 break;
810 default:
811
812
813
814 dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", val);
815 return -EINVAL;
816 }
817
818 ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
819 ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
820 ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
821
822 ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
823 ndev->limits.msix_cnt = SNB_MSIX_CNT;
824 ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
825
826 return 0;
827}
828
829static int ntb_bwd_setup(struct ntb_device *ndev)
830{
831 int rc;
832 u32 val;
833
834 ndev->hw_type = BWD_HW;
835
836 rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
837 if (rc)
838 return rc;
839
840 switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
841 case NTB_CONN_B2B:
842 ndev->conn_type = NTB_CONN_B2B;
843 break;
844 case NTB_CONN_RP:
845 default:
846 dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
847 return -EINVAL;
848 }
849
850 if (val & BWD_PPD_DEV_TYPE)
851 ndev->dev_type = NTB_DEV_DSD;
852 else
853 ndev->dev_type = NTB_DEV_USD;
854
855
856 rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
857 val | BWD_PPD_INIT_LINK);
858 if (rc)
859 return rc;
860
861 ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
862 ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
863 ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
864 ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
865 ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
866 ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
867 ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
868 ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
869 ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
870 ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
871 ndev->limits.max_mw = BWD_MAX_MW;
872 ndev->limits.max_spads = BWD_MAX_SPADS;
873 ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
874 ndev->limits.msix_cnt = BWD_MSIX_CNT;
875 ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
876
877
878 INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
879 INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
880 schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
881
882 return 0;
883}
884
885static int ntb_device_setup(struct ntb_device *ndev)
886{
887 int rc;
888
889 switch (ndev->pdev->device) {
890 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
891 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
892 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
893 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
894 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
895 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
896 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
897 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
898 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
899 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
900 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
901 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
902 rc = ntb_xeon_setup(ndev);
903 break;
904 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
905 rc = ntb_bwd_setup(ndev);
906 break;
907 default:
908 rc = -ENODEV;
909 }
910
911 if (rc)
912 return rc;
913
914 dev_info(&ndev->pdev->dev, "Device Type = %s\n",
915 ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
916
917 if (ndev->conn_type == NTB_CONN_B2B)
918
919 writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
920 ndev->reg_ofs.spci_cmd);
921
922 return 0;
923}
924
925static void ntb_device_free(struct ntb_device *ndev)
926{
927 if (ndev->hw_type == BWD_HW) {
928 cancel_delayed_work_sync(&ndev->hb_timer);
929 cancel_delayed_work_sync(&ndev->lr_timer);
930 }
931}
932
933static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
934{
935 struct ntb_db_cb *db_cb = data;
936 struct ntb_device *ndev = db_cb->ndev;
937
938 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
939 db_cb->db_num);
940
941 if (db_cb->callback)
942 db_cb->callback(db_cb->data, db_cb->db_num);
943
944
945
946
947 ndev->last_ts = jiffies;
948
949 writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
950
951 return IRQ_HANDLED;
952}
953
954static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
955{
956 struct ntb_db_cb *db_cb = data;
957 struct ntb_device *ndev = db_cb->ndev;
958
959 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
960 db_cb->db_num);
961
962 if (db_cb->callback)
963 db_cb->callback(db_cb->data, db_cb->db_num);
964
965
966
967
968
969
970 writew(((1 << ndev->bits_per_vector) - 1) <<
971 (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
972
973 return IRQ_HANDLED;
974}
975
976
977static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
978{
979 struct ntb_device *ndev = dev;
980 int rc;
981
982 dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
983
984 rc = ntb_link_status(ndev);
985 if (rc)
986 dev_err(&ndev->pdev->dev, "Error determining link status\n");
987
988
989 writew(1 << ndev->limits.max_db_bits, ndev->reg_ofs.ldb);
990
991 return IRQ_HANDLED;
992}
993
994static irqreturn_t ntb_interrupt(int irq, void *dev)
995{
996 struct ntb_device *ndev = dev;
997 unsigned int i = 0;
998
999 if (ndev->hw_type == BWD_HW) {
1000 u64 ldb = readq(ndev->reg_ofs.ldb);
1001
1002 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
1003
1004 while (ldb) {
1005 i = __ffs(ldb);
1006 ldb &= ldb - 1;
1007 bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
1008 }
1009 } else {
1010 u16 ldb = readw(ndev->reg_ofs.ldb);
1011
1012 dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
1013
1014 if (ldb & SNB_DB_HW_LINK) {
1015 xeon_event_msix_irq(irq, dev);
1016 ldb &= ~SNB_DB_HW_LINK;
1017 }
1018
1019 while (ldb) {
1020 i = __ffs(ldb);
1021 ldb &= ldb - 1;
1022 xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
1023 }
1024 }
1025
1026 return IRQ_HANDLED;
1027}
1028
1029static int ntb_setup_msix(struct ntb_device *ndev)
1030{
1031 struct pci_dev *pdev = ndev->pdev;
1032 struct msix_entry *msix;
1033 int msix_entries;
1034 int rc, i;
1035 u16 val;
1036
1037 if (!pdev->msix_cap) {
1038 rc = -EIO;
1039 goto err;
1040 }
1041
1042 rc = pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &val);
1043 if (rc)
1044 goto err;
1045
1046 msix_entries = msix_table_size(val);
1047 if (msix_entries > ndev->limits.msix_cnt) {
1048 rc = -EINVAL;
1049 goto err;
1050 }
1051
1052 ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
1053 GFP_KERNEL);
1054 if (!ndev->msix_entries) {
1055 rc = -ENOMEM;
1056 goto err;
1057 }
1058
1059 for (i = 0; i < msix_entries; i++)
1060 ndev->msix_entries[i].entry = i;
1061
1062 rc = pci_enable_msix(pdev, ndev->msix_entries, msix_entries);
1063 if (rc < 0)
1064 goto err1;
1065 if (rc > 0) {
1066
1067
1068
1069 if (ndev->hw_type != BWD_HW) {
1070 rc = -EIO;
1071 goto err1;
1072 }
1073
1074 dev_warn(&pdev->dev,
1075 "Only %d MSI-X vectors. Limiting the number of queues to that number.\n",
1076 rc);
1077 msix_entries = rc;
1078 }
1079
1080 for (i = 0; i < msix_entries; i++) {
1081 msix = &ndev->msix_entries[i];
1082 WARN_ON(!msix->vector);
1083
1084
1085 if (ndev->hw_type == BWD_HW) {
1086 rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
1087 "ntb-callback-msix", &ndev->db_cb[i]);
1088 if (rc)
1089 goto err2;
1090 } else {
1091 if (i == msix_entries - 1) {
1092 rc = request_irq(msix->vector,
1093 xeon_event_msix_irq, 0,
1094 "ntb-event-msix", ndev);
1095 if (rc)
1096 goto err2;
1097 } else {
1098 rc = request_irq(msix->vector,
1099 xeon_callback_msix_irq, 0,
1100 "ntb-callback-msix",
1101 &ndev->db_cb[i]);
1102 if (rc)
1103 goto err2;
1104 }
1105 }
1106 }
1107
1108 ndev->num_msix = msix_entries;
1109 if (ndev->hw_type == BWD_HW)
1110 ndev->max_cbs = msix_entries;
1111 else
1112 ndev->max_cbs = msix_entries - 1;
1113
1114 return 0;
1115
1116err2:
1117 while (--i >= 0) {
1118 msix = &ndev->msix_entries[i];
1119 if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
1120 free_irq(msix->vector, ndev);
1121 else
1122 free_irq(msix->vector, &ndev->db_cb[i]);
1123 }
1124 pci_disable_msix(pdev);
1125err1:
1126 kfree(ndev->msix_entries);
1127 dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
1128err:
1129 ndev->num_msix = 0;
1130 return rc;
1131}
1132
1133static int ntb_setup_msi(struct ntb_device *ndev)
1134{
1135 struct pci_dev *pdev = ndev->pdev;
1136 int rc;
1137
1138 rc = pci_enable_msi(pdev);
1139 if (rc)
1140 return rc;
1141
1142 rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
1143 if (rc) {
1144 pci_disable_msi(pdev);
1145 dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
1146 return rc;
1147 }
1148
1149 return 0;
1150}
1151
1152static int ntb_setup_intx(struct ntb_device *ndev)
1153{
1154 struct pci_dev *pdev = ndev->pdev;
1155 int rc;
1156
1157 pci_msi_off(pdev);
1158
1159
1160 pci_intx(pdev, 1);
1161
1162 rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
1163 ndev);
1164 if (rc)
1165 return rc;
1166
1167 return 0;
1168}
1169
1170static int ntb_setup_interrupts(struct ntb_device *ndev)
1171{
1172 int rc;
1173
1174
1175
1176
1177 if (ndev->hw_type == BWD_HW)
1178 writeq(~0, ndev->reg_ofs.ldb_mask);
1179 else
1180 writew(~(1 << ndev->limits.max_db_bits),
1181 ndev->reg_ofs.ldb_mask);
1182
1183 rc = ntb_setup_msix(ndev);
1184 if (!rc)
1185 goto done;
1186
1187 ndev->bits_per_vector = 1;
1188 ndev->max_cbs = ndev->limits.max_db_bits;
1189
1190 rc = ntb_setup_msi(ndev);
1191 if (!rc)
1192 goto done;
1193
1194 rc = ntb_setup_intx(ndev);
1195 if (rc) {
1196 dev_err(&ndev->pdev->dev, "no usable interrupts\n");
1197 return rc;
1198 }
1199
1200done:
1201 return 0;
1202}
1203
1204static void ntb_free_interrupts(struct ntb_device *ndev)
1205{
1206 struct pci_dev *pdev = ndev->pdev;
1207
1208
1209 if (ndev->hw_type == BWD_HW)
1210 writeq(~0, ndev->reg_ofs.ldb_mask);
1211 else
1212 writew(~0, ndev->reg_ofs.ldb_mask);
1213
1214 if (ndev->num_msix) {
1215 struct msix_entry *msix;
1216 u32 i;
1217
1218 for (i = 0; i < ndev->num_msix; i++) {
1219 msix = &ndev->msix_entries[i];
1220 if (ndev->hw_type != BWD_HW && i == ndev->num_msix - 1)
1221 free_irq(msix->vector, ndev);
1222 else
1223 free_irq(msix->vector, &ndev->db_cb[i]);
1224 }
1225 pci_disable_msix(pdev);
1226 } else {
1227 free_irq(pdev->irq, ndev);
1228
1229 if (pci_dev_msi_enabled(pdev))
1230 pci_disable_msi(pdev);
1231 }
1232}
1233
1234static int ntb_create_callbacks(struct ntb_device *ndev)
1235{
1236 int i;
1237
1238
1239
1240
1241
1242
1243 ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
1244 sizeof(struct ntb_db_cb),
1245 GFP_KERNEL);
1246 if (!ndev->db_cb)
1247 return -ENOMEM;
1248
1249 for (i = 0; i < ndev->limits.max_db_bits; i++) {
1250 ndev->db_cb[i].db_num = i;
1251 ndev->db_cb[i].ndev = ndev;
1252 }
1253
1254 return 0;
1255}
1256
1257static void ntb_free_callbacks(struct ntb_device *ndev)
1258{
1259 int i;
1260
1261 for (i = 0; i < ndev->limits.max_db_bits; i++)
1262 ntb_unregister_db_callback(ndev, i);
1263
1264 kfree(ndev->db_cb);
1265}
1266
1267static void ntb_setup_debugfs(struct ntb_device *ndev)
1268{
1269 if (!debugfs_initialized())
1270 return;
1271
1272 if (!debugfs_dir)
1273 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1274
1275 ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
1276 debugfs_dir);
1277}
1278
1279static void ntb_free_debugfs(struct ntb_device *ndev)
1280{
1281 debugfs_remove_recursive(ndev->debugfs_dir);
1282
1283 if (debugfs_dir && simple_empty(debugfs_dir)) {
1284 debugfs_remove_recursive(debugfs_dir);
1285 debugfs_dir = NULL;
1286 }
1287}
1288
1289static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1290{
1291 struct ntb_device *ndev;
1292 int rc, i;
1293
1294 ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
1295 if (!ndev)
1296 return -ENOMEM;
1297
1298 ndev->pdev = pdev;
1299 ndev->link_status = NTB_LINK_DOWN;
1300 pci_set_drvdata(pdev, ndev);
1301 ntb_setup_debugfs(ndev);
1302
1303 rc = pci_enable_device(pdev);
1304 if (rc)
1305 goto err;
1306
1307 pci_set_master(ndev->pdev);
1308
1309 rc = pci_request_selected_regions(pdev, NTB_BAR_MASK, KBUILD_MODNAME);
1310 if (rc)
1311 goto err1;
1312
1313 ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
1314 if (!ndev->reg_base) {
1315 dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
1316 rc = -EIO;
1317 goto err2;
1318 }
1319
1320 for (i = 0; i < NTB_MAX_NUM_MW; i++) {
1321 ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
1322 ndev->mw[i].vbase =
1323 ioremap_wc(pci_resource_start(pdev, MW_TO_BAR(i)),
1324 ndev->mw[i].bar_sz);
1325 dev_info(&pdev->dev, "MW %d size %llu\n", i,
1326 (unsigned long long) ndev->mw[i].bar_sz);
1327 if (!ndev->mw[i].vbase) {
1328 dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
1329 MW_TO_BAR(i));
1330 rc = -EIO;
1331 goto err3;
1332 }
1333 }
1334
1335 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1336 if (rc) {
1337 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1338 if (rc)
1339 goto err3;
1340
1341 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1342 }
1343
1344 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1345 if (rc) {
1346 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1347 if (rc)
1348 goto err3;
1349
1350 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1351 }
1352
1353 rc = ntb_device_setup(ndev);
1354 if (rc)
1355 goto err3;
1356
1357 rc = ntb_create_callbacks(ndev);
1358 if (rc)
1359 goto err4;
1360
1361 rc = ntb_setup_interrupts(ndev);
1362 if (rc)
1363 goto err5;
1364
1365
1366
1367
1368 for (i = 0; i < ndev->limits.max_spads; i++) {
1369 ntb_write_local_spad(ndev, i, 0);
1370 ntb_write_remote_spad(ndev, i, 0);
1371 }
1372
1373 rc = ntb_transport_init(pdev);
1374 if (rc)
1375 goto err6;
1376
1377
1378 writel(NTB_CNTL_BAR23_SNOOP | NTB_CNTL_BAR45_SNOOP,
1379 ndev->reg_ofs.lnk_cntl);
1380
1381 return 0;
1382
1383err6:
1384 ntb_free_interrupts(ndev);
1385err5:
1386 ntb_free_callbacks(ndev);
1387err4:
1388 ntb_device_free(ndev);
1389err3:
1390 for (i--; i >= 0; i--)
1391 iounmap(ndev->mw[i].vbase);
1392 iounmap(ndev->reg_base);
1393err2:
1394 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1395err1:
1396 pci_disable_device(pdev);
1397err:
1398 ntb_free_debugfs(ndev);
1399 kfree(ndev);
1400
1401 dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
1402 return rc;
1403}
1404
1405static void ntb_pci_remove(struct pci_dev *pdev)
1406{
1407 struct ntb_device *ndev = pci_get_drvdata(pdev);
1408 int i;
1409 u32 ntb_cntl;
1410
1411
1412 ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
1413 ntb_cntl |= NTB_CNTL_LINK_DISABLE;
1414 writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
1415
1416 ntb_transport_free(ndev->ntb_transport);
1417
1418 ntb_free_interrupts(ndev);
1419 ntb_free_callbacks(ndev);
1420 ntb_device_free(ndev);
1421
1422 for (i = 0; i < NTB_MAX_NUM_MW; i++)
1423 iounmap(ndev->mw[i].vbase);
1424
1425 iounmap(ndev->reg_base);
1426 pci_release_selected_regions(pdev, NTB_BAR_MASK);
1427 pci_disable_device(pdev);
1428 ntb_free_debugfs(ndev);
1429 kfree(ndev);
1430}
1431
1432static struct pci_driver ntb_pci_driver = {
1433 .name = KBUILD_MODNAME,
1434 .id_table = ntb_pci_tbl,
1435 .probe = ntb_pci_probe,
1436 .remove = ntb_pci_remove,
1437};
1438module_pci_driver(ntb_pci_driver);
1439