1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50#include <linux/debugfs.h>
51#include <linux/delay.h>
52#include <linux/init.h>
53#include <linux/interrupt.h>
54#include <linux/module.h>
55#include <linux/pci.h>
56#include <linux/random.h>
57#include <linux/slab.h>
58#include <linux/ntb.h>
59
60#include "ntb_hw_intel.h"
61#include "ntb_hw_gen1.h"
62#include "ntb_hw_gen3.h"
63
64#define NTB_NAME "ntb_hw_intel"
65#define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
66#define NTB_VER "2.0"
67
68MODULE_DESCRIPTION(NTB_DESC);
69MODULE_VERSION(NTB_VER);
70MODULE_LICENSE("Dual BSD/GPL");
71MODULE_AUTHOR("Intel Corporation");
72
73#define bar0_off(base, bar) ((base) + ((bar) << 2))
74#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
75
76static const struct intel_ntb_reg xeon_reg;
77static const struct intel_ntb_alt_reg xeon_pri_reg;
78static const struct intel_ntb_alt_reg xeon_sec_reg;
79static const struct intel_ntb_alt_reg xeon_b2b_reg;
80static const struct intel_ntb_xlat_reg xeon_pri_xlat;
81static const struct intel_ntb_xlat_reg xeon_sec_xlat;
82static const struct ntb_dev_ops intel_ntb_ops;
83
84static const struct file_operations intel_ntb_debugfs_info;
85static struct dentry *debugfs_dir;
86
87static int b2b_mw_idx = -1;
88module_param(b2b_mw_idx, int, 0644);
89MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
90 "value of zero or positive starts from first mw idx, and a "
91 "negative value starts from last mw idx. Both sides MUST "
92 "set the same value here!");
93
94static unsigned int b2b_mw_share;
95module_param(b2b_mw_share, uint, 0644);
96MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
97 "ntb so that the peer ntb only occupies the first half of "
98 "the mw, so the second half can still be used as a mw. Both "
99 "sides MUST set the same value here!");
100
101module_param_named(xeon_b2b_usd_bar2_addr64,
102 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
103MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
104 "XEON B2B USD BAR 2 64-bit address");
105
106module_param_named(xeon_b2b_usd_bar4_addr64,
107 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
108MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
109 "XEON B2B USD BAR 4 64-bit address");
110
111module_param_named(xeon_b2b_usd_bar4_addr32,
112 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
113MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
114 "XEON B2B USD split-BAR 4 32-bit address");
115
116module_param_named(xeon_b2b_usd_bar5_addr32,
117 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
118MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
119 "XEON B2B USD split-BAR 5 32-bit address");
120
121module_param_named(xeon_b2b_dsd_bar2_addr64,
122 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
123MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
124 "XEON B2B DSD BAR 2 64-bit address");
125
126module_param_named(xeon_b2b_dsd_bar4_addr64,
127 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
128MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
129 "XEON B2B DSD BAR 4 64-bit address");
130
131module_param_named(xeon_b2b_dsd_bar4_addr32,
132 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
133MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
134 "XEON B2B DSD split-BAR 4 32-bit address");
135
136module_param_named(xeon_b2b_dsd_bar5_addr32,
137 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
138MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
139 "XEON B2B DSD split-BAR 5 32-bit address");
140
141
142static int xeon_init_isr(struct intel_ntb_dev *ndev);
143
144static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
145{
146 ndev->unsafe_flags = 0;
147 ndev->unsafe_flags_ignore = 0;
148
149
150 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
151 if (!ntb_topo_is_b2b(ndev->ntb.topo))
152 ndev->unsafe_flags |= NTB_UNSAFE_DB;
153
154
155 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
156 ndev->unsafe_flags |= NTB_UNSAFE_DB;
157 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
158 }
159}
160
161static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
162 unsigned long flag)
163{
164 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
165}
166
167static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
168 unsigned long flag)
169{
170 flag &= ndev->unsafe_flags;
171 ndev->unsafe_flags_ignore |= flag;
172
173 return !!flag;
174}
175
176int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
177{
178 if (idx < 0 || idx >= ndev->mw_count)
179 return -EINVAL;
180 return ndev->reg->mw_bar[idx];
181}
182
183void ndev_db_addr(struct intel_ntb_dev *ndev,
184 phys_addr_t *db_addr, resource_size_t *db_size,
185 phys_addr_t reg_addr, unsigned long reg)
186{
187 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
188 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
189
190 if (db_addr) {
191 *db_addr = reg_addr + reg;
192 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
193 }
194
195 if (db_size) {
196 *db_size = ndev->reg->db_size;
197 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
198 }
199}
200
201u64 ndev_db_read(struct intel_ntb_dev *ndev,
202 void __iomem *mmio)
203{
204 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
205 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
206
207 return ndev->reg->db_ioread(mmio);
208}
209
210int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
211 void __iomem *mmio)
212{
213 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
214 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
215
216 if (db_bits & ~ndev->db_valid_mask)
217 return -EINVAL;
218
219 ndev->reg->db_iowrite(db_bits, mmio);
220
221 return 0;
222}
223
224static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
225 void __iomem *mmio)
226{
227 unsigned long irqflags;
228
229 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
230 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
231
232 if (db_bits & ~ndev->db_valid_mask)
233 return -EINVAL;
234
235 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
236 {
237 ndev->db_mask |= db_bits;
238 ndev->reg->db_iowrite(ndev->db_mask, mmio);
239 }
240 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
241
242 return 0;
243}
244
245static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
246 void __iomem *mmio)
247{
248 unsigned long irqflags;
249
250 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
251 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
252
253 if (db_bits & ~ndev->db_valid_mask)
254 return -EINVAL;
255
256 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
257 {
258 ndev->db_mask &= ~db_bits;
259 ndev->reg->db_iowrite(ndev->db_mask, mmio);
260 }
261 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
262
263 return 0;
264}
265
266static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
267{
268 u64 shift, mask;
269
270 shift = ndev->db_vec_shift;
271 mask = BIT_ULL(shift) - 1;
272
273 return mask << (shift * db_vector);
274}
275
276static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
277 phys_addr_t *spad_addr, phys_addr_t reg_addr,
278 unsigned long reg)
279{
280 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
281 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
282
283 if (idx < 0 || idx >= ndev->spad_count)
284 return -EINVAL;
285
286 if (spad_addr) {
287 *spad_addr = reg_addr + reg + (idx << 2);
288 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
289 *spad_addr);
290 }
291
292 return 0;
293}
294
295static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
296 void __iomem *mmio)
297{
298 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
299 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
300
301 if (idx < 0 || idx >= ndev->spad_count)
302 return 0;
303
304 return ioread32(mmio + (idx << 2));
305}
306
307static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
308 void __iomem *mmio)
309{
310 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
311 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
312
313 if (idx < 0 || idx >= ndev->spad_count)
314 return -EINVAL;
315
316 iowrite32(val, mmio + (idx << 2));
317
318 return 0;
319}
320
321static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
322{
323 u64 vec_mask;
324
325 vec_mask = ndev_vec_mask(ndev, vec);
326
327 if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
328 vec_mask |= ndev->db_link_mask;
329
330 dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
331
332 ndev->last_ts = jiffies;
333
334 if (vec_mask & ndev->db_link_mask) {
335 if (ndev->reg->poll_link(ndev))
336 ntb_link_event(&ndev->ntb);
337 }
338
339 if (vec_mask & ndev->db_valid_mask)
340 ntb_db_event(&ndev->ntb, vec);
341
342 return IRQ_HANDLED;
343}
344
345static irqreturn_t ndev_vec_isr(int irq, void *dev)
346{
347 struct intel_ntb_vec *nvec = dev;
348
349 dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d nvec->num: %d\n",
350 irq, nvec->num);
351
352 return ndev_interrupt(nvec->ndev, nvec->num);
353}
354
355static irqreturn_t ndev_irq_isr(int irq, void *dev)
356{
357 struct intel_ntb_dev *ndev = dev;
358
359 return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
360}
361
362int ndev_init_isr(struct intel_ntb_dev *ndev,
363 int msix_min, int msix_max,
364 int msix_shift, int total_shift)
365{
366 struct pci_dev *pdev;
367 int rc, i, msix_count, node;
368
369 pdev = ndev->ntb.pdev;
370
371 node = dev_to_node(&pdev->dev);
372
373
374 ndev->db_mask = ndev->db_valid_mask;
375 ndev->reg->db_iowrite(ndev->db_mask,
376 ndev->self_mmio +
377 ndev->self_reg->db_mask);
378
379
380
381 ndev->vec = kcalloc_node(msix_max, sizeof(*ndev->vec),
382 GFP_KERNEL, node);
383 if (!ndev->vec)
384 goto err_msix_vec_alloc;
385
386 ndev->msix = kcalloc_node(msix_max, sizeof(*ndev->msix),
387 GFP_KERNEL, node);
388 if (!ndev->msix)
389 goto err_msix_alloc;
390
391 for (i = 0; i < msix_max; ++i)
392 ndev->msix[i].entry = i;
393
394 msix_count = pci_enable_msix_range(pdev, ndev->msix,
395 msix_min, msix_max);
396 if (msix_count < 0)
397 goto err_msix_enable;
398
399 for (i = 0; i < msix_count; ++i) {
400 ndev->vec[i].ndev = ndev;
401 ndev->vec[i].num = i;
402 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
403 "ndev_vec_isr", &ndev->vec[i]);
404 if (rc)
405 goto err_msix_request;
406 }
407
408 dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
409 ndev->db_vec_count = msix_count;
410 ndev->db_vec_shift = msix_shift;
411 return 0;
412
413err_msix_request:
414 while (i-- > 0)
415 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
416 pci_disable_msix(pdev);
417err_msix_enable:
418 kfree(ndev->msix);
419err_msix_alloc:
420 kfree(ndev->vec);
421err_msix_vec_alloc:
422 ndev->msix = NULL;
423 ndev->vec = NULL;
424
425
426
427 rc = pci_enable_msi(pdev);
428 if (rc)
429 goto err_msi_enable;
430
431 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
432 "ndev_irq_isr", ndev);
433 if (rc)
434 goto err_msi_request;
435
436 dev_dbg(&pdev->dev, "Using msi interrupts\n");
437 ndev->db_vec_count = 1;
438 ndev->db_vec_shift = total_shift;
439 return 0;
440
441err_msi_request:
442 pci_disable_msi(pdev);
443err_msi_enable:
444
445
446
447 pci_intx(pdev, 1);
448
449 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
450 "ndev_irq_isr", ndev);
451 if (rc)
452 goto err_intx_request;
453
454 dev_dbg(&pdev->dev, "Using intx interrupts\n");
455 ndev->db_vec_count = 1;
456 ndev->db_vec_shift = total_shift;
457 return 0;
458
459err_intx_request:
460 return rc;
461}
462
463static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
464{
465 struct pci_dev *pdev;
466 int i;
467
468 pdev = ndev->ntb.pdev;
469
470
471 ndev->db_mask = ndev->db_valid_mask;
472 ndev->reg->db_iowrite(ndev->db_mask,
473 ndev->self_mmio +
474 ndev->self_reg->db_mask);
475
476 if (ndev->msix) {
477 i = ndev->db_vec_count;
478 while (i--)
479 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
480 pci_disable_msix(pdev);
481 kfree(ndev->msix);
482 kfree(ndev->vec);
483 } else {
484 free_irq(pdev->irq, ndev);
485 if (pci_dev_msi_enabled(pdev))
486 pci_disable_msi(pdev);
487 }
488}
489
490static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
491 size_t count, loff_t *offp)
492{
493 struct intel_ntb_dev *ndev;
494 struct pci_dev *pdev;
495 void __iomem *mmio;
496 char *buf;
497 size_t buf_size;
498 ssize_t ret, off;
499 union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
500
501 ndev = filp->private_data;
502 pdev = ndev->ntb.pdev;
503 mmio = ndev->self_mmio;
504
505 buf_size = min(count, 0x800ul);
506
507 buf = kmalloc(buf_size, GFP_KERNEL);
508 if (!buf)
509 return -ENOMEM;
510
511 off = 0;
512
513 off += scnprintf(buf + off, buf_size - off,
514 "NTB Device Information:\n");
515
516 off += scnprintf(buf + off, buf_size - off,
517 "Connection Topology -\t%s\n",
518 ntb_topo_string(ndev->ntb.topo));
519
520 if (ndev->b2b_idx != UINT_MAX) {
521 off += scnprintf(buf + off, buf_size - off,
522 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
523 off += scnprintf(buf + off, buf_size - off,
524 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
525 }
526
527 off += scnprintf(buf + off, buf_size - off,
528 "BAR4 Split -\t\t%s\n",
529 ndev->bar4_split ? "yes" : "no");
530
531 off += scnprintf(buf + off, buf_size - off,
532 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
533 off += scnprintf(buf + off, buf_size - off,
534 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
535
536 if (!ndev->reg->link_is_up(ndev)) {
537 off += scnprintf(buf + off, buf_size - off,
538 "Link Status -\t\tDown\n");
539 } else {
540 off += scnprintf(buf + off, buf_size - off,
541 "Link Status -\t\tUp\n");
542 off += scnprintf(buf + off, buf_size - off,
543 "Link Speed -\t\tPCI-E Gen %u\n",
544 NTB_LNK_STA_SPEED(ndev->lnk_sta));
545 off += scnprintf(buf + off, buf_size - off,
546 "Link Width -\t\tx%u\n",
547 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
548 }
549
550 off += scnprintf(buf + off, buf_size - off,
551 "Memory Window Count -\t%u\n", ndev->mw_count);
552 off += scnprintf(buf + off, buf_size - off,
553 "Scratchpad Count -\t%u\n", ndev->spad_count);
554 off += scnprintf(buf + off, buf_size - off,
555 "Doorbell Count -\t%u\n", ndev->db_count);
556 off += scnprintf(buf + off, buf_size - off,
557 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
558 off += scnprintf(buf + off, buf_size - off,
559 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
560
561 off += scnprintf(buf + off, buf_size - off,
562 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
563 off += scnprintf(buf + off, buf_size - off,
564 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
565 off += scnprintf(buf + off, buf_size - off,
566 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
567
568 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
569 off += scnprintf(buf + off, buf_size - off,
570 "Doorbell Mask -\t\t%#llx\n", u.v64);
571
572 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
573 off += scnprintf(buf + off, buf_size - off,
574 "Doorbell Bell -\t\t%#llx\n", u.v64);
575
576 off += scnprintf(buf + off, buf_size - off,
577 "\nNTB Window Size:\n");
578
579 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
580 off += scnprintf(buf + off, buf_size - off,
581 "PBAR23SZ %hhu\n", u.v8);
582 if (!ndev->bar4_split) {
583 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
584 off += scnprintf(buf + off, buf_size - off,
585 "PBAR45SZ %hhu\n", u.v8);
586 } else {
587 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
588 off += scnprintf(buf + off, buf_size - off,
589 "PBAR4SZ %hhu\n", u.v8);
590 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
591 off += scnprintf(buf + off, buf_size - off,
592 "PBAR5SZ %hhu\n", u.v8);
593 }
594
595 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
596 off += scnprintf(buf + off, buf_size - off,
597 "SBAR23SZ %hhu\n", u.v8);
598 if (!ndev->bar4_split) {
599 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
600 off += scnprintf(buf + off, buf_size - off,
601 "SBAR45SZ %hhu\n", u.v8);
602 } else {
603 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
604 off += scnprintf(buf + off, buf_size - off,
605 "SBAR4SZ %hhu\n", u.v8);
606 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
607 off += scnprintf(buf + off, buf_size - off,
608 "SBAR5SZ %hhu\n", u.v8);
609 }
610
611 off += scnprintf(buf + off, buf_size - off,
612 "\nNTB Incoming XLAT:\n");
613
614 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
615 off += scnprintf(buf + off, buf_size - off,
616 "XLAT23 -\t\t%#018llx\n", u.v64);
617
618 if (ndev->bar4_split) {
619 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
620 off += scnprintf(buf + off, buf_size - off,
621 "XLAT4 -\t\t\t%#06x\n", u.v32);
622
623 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
624 off += scnprintf(buf + off, buf_size - off,
625 "XLAT5 -\t\t\t%#06x\n", u.v32);
626 } else {
627 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
628 off += scnprintf(buf + off, buf_size - off,
629 "XLAT45 -\t\t%#018llx\n", u.v64);
630 }
631
632 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
633 off += scnprintf(buf + off, buf_size - off,
634 "LMT23 -\t\t\t%#018llx\n", u.v64);
635
636 if (ndev->bar4_split) {
637 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
638 off += scnprintf(buf + off, buf_size - off,
639 "LMT4 -\t\t\t%#06x\n", u.v32);
640 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
641 off += scnprintf(buf + off, buf_size - off,
642 "LMT5 -\t\t\t%#06x\n", u.v32);
643 } else {
644 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
645 off += scnprintf(buf + off, buf_size - off,
646 "LMT45 -\t\t\t%#018llx\n", u.v64);
647 }
648
649 if (pdev_is_gen1(pdev)) {
650 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
651 off += scnprintf(buf + off, buf_size - off,
652 "\nNTB Outgoing B2B XLAT:\n");
653
654 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
655 off += scnprintf(buf + off, buf_size - off,
656 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
657
658 if (ndev->bar4_split) {
659 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
660 off += scnprintf(buf + off, buf_size - off,
661 "B2B XLAT4 -\t\t%#06x\n",
662 u.v32);
663 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
664 off += scnprintf(buf + off, buf_size - off,
665 "B2B XLAT5 -\t\t%#06x\n",
666 u.v32);
667 } else {
668 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
669 off += scnprintf(buf + off, buf_size - off,
670 "B2B XLAT45 -\t\t%#018llx\n",
671 u.v64);
672 }
673
674 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
675 off += scnprintf(buf + off, buf_size - off,
676 "B2B LMT23 -\t\t%#018llx\n", u.v64);
677
678 if (ndev->bar4_split) {
679 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
680 off += scnprintf(buf + off, buf_size - off,
681 "B2B LMT4 -\t\t%#06x\n",
682 u.v32);
683 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
684 off += scnprintf(buf + off, buf_size - off,
685 "B2B LMT5 -\t\t%#06x\n",
686 u.v32);
687 } else {
688 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
689 off += scnprintf(buf + off, buf_size - off,
690 "B2B LMT45 -\t\t%#018llx\n",
691 u.v64);
692 }
693
694 off += scnprintf(buf + off, buf_size - off,
695 "\nNTB Secondary BAR:\n");
696
697 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
698 off += scnprintf(buf + off, buf_size - off,
699 "SBAR01 -\t\t%#018llx\n", u.v64);
700
701 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
702 off += scnprintf(buf + off, buf_size - off,
703 "SBAR23 -\t\t%#018llx\n", u.v64);
704
705 if (ndev->bar4_split) {
706 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
707 off += scnprintf(buf + off, buf_size - off,
708 "SBAR4 -\t\t\t%#06x\n", u.v32);
709 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
710 off += scnprintf(buf + off, buf_size - off,
711 "SBAR5 -\t\t\t%#06x\n", u.v32);
712 } else {
713 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
714 off += scnprintf(buf + off, buf_size - off,
715 "SBAR45 -\t\t%#018llx\n",
716 u.v64);
717 }
718 }
719
720 off += scnprintf(buf + off, buf_size - off,
721 "\nXEON NTB Statistics:\n");
722
723 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
724 off += scnprintf(buf + off, buf_size - off,
725 "Upstream Memory Miss -\t%u\n", u.v16);
726
727 off += scnprintf(buf + off, buf_size - off,
728 "\nXEON NTB Hardware Errors:\n");
729
730 if (!pci_read_config_word(pdev,
731 XEON_DEVSTS_OFFSET, &u.v16))
732 off += scnprintf(buf + off, buf_size - off,
733 "DEVSTS -\t\t%#06x\n", u.v16);
734
735 if (!pci_read_config_word(pdev,
736 XEON_LINK_STATUS_OFFSET, &u.v16))
737 off += scnprintf(buf + off, buf_size - off,
738 "LNKSTS -\t\t%#06x\n", u.v16);
739
740 if (!pci_read_config_dword(pdev,
741 XEON_UNCERRSTS_OFFSET, &u.v32))
742 off += scnprintf(buf + off, buf_size - off,
743 "UNCERRSTS -\t\t%#06x\n", u.v32);
744
745 if (!pci_read_config_dword(pdev,
746 XEON_CORERRSTS_OFFSET, &u.v32))
747 off += scnprintf(buf + off, buf_size - off,
748 "CORERRSTS -\t\t%#06x\n", u.v32);
749 }
750
751 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
752 kfree(buf);
753 return ret;
754}
755
756static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
757 size_t count, loff_t *offp)
758{
759 struct intel_ntb_dev *ndev = filp->private_data;
760
761 if (pdev_is_gen1(ndev->ntb.pdev))
762 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
763 else if (pdev_is_gen3(ndev->ntb.pdev))
764 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
765
766 return -ENXIO;
767}
768
769static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
770{
771 if (!debugfs_dir) {
772 ndev->debugfs_dir = NULL;
773 ndev->debugfs_info = NULL;
774 } else {
775 ndev->debugfs_dir =
776 debugfs_create_dir(pci_name(ndev->ntb.pdev),
777 debugfs_dir);
778 if (!ndev->debugfs_dir)
779 ndev->debugfs_info = NULL;
780 else
781 ndev->debugfs_info =
782 debugfs_create_file("info", S_IRUSR,
783 ndev->debugfs_dir, ndev,
784 &intel_ntb_debugfs_info);
785 }
786}
787
788static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
789{
790 debugfs_remove_recursive(ndev->debugfs_dir);
791}
792
793int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
794{
795 if (pidx != NTB_DEF_PEER_IDX)
796 return -EINVAL;
797
798 return ntb_ndev(ntb)->mw_count;
799}
800
801int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
802 resource_size_t *addr_align,
803 resource_size_t *size_align,
804 resource_size_t *size_max)
805{
806 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
807 resource_size_t bar_size, mw_size;
808 int bar;
809
810 if (pidx != NTB_DEF_PEER_IDX)
811 return -EINVAL;
812
813 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
814 idx += 1;
815
816 bar = ndev_mw_to_bar(ndev, idx);
817 if (bar < 0)
818 return bar;
819
820 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
821
822 if (idx == ndev->b2b_idx)
823 mw_size = bar_size - ndev->b2b_off;
824 else
825 mw_size = bar_size;
826
827 if (addr_align)
828 *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
829
830 if (size_align)
831 *size_align = 1;
832
833 if (size_max)
834 *size_max = mw_size;
835
836 return 0;
837}
838
839static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
840 dma_addr_t addr, resource_size_t size)
841{
842 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
843 unsigned long base_reg, xlat_reg, limit_reg;
844 resource_size_t bar_size, mw_size;
845 void __iomem *mmio;
846 u64 base, limit, reg_val;
847 int bar;
848
849 if (pidx != NTB_DEF_PEER_IDX)
850 return -EINVAL;
851
852 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
853 idx += 1;
854
855 bar = ndev_mw_to_bar(ndev, idx);
856 if (bar < 0)
857 return bar;
858
859 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
860
861 if (idx == ndev->b2b_idx)
862 mw_size = bar_size - ndev->b2b_off;
863 else
864 mw_size = bar_size;
865
866
867 if (addr & (bar_size - 1))
868 return -EINVAL;
869
870
871 if (size > mw_size)
872 return -EINVAL;
873
874 mmio = ndev->self_mmio;
875 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
876 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
877 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
878
879 if (bar < 4 || !ndev->bar4_split) {
880 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
881
882
883 if (limit_reg && size != mw_size)
884 limit = base + size;
885 else
886 limit = 0;
887
888
889 iowrite64(addr, mmio + xlat_reg);
890 reg_val = ioread64(mmio + xlat_reg);
891 if (reg_val != addr) {
892 iowrite64(0, mmio + xlat_reg);
893 return -EIO;
894 }
895
896
897 iowrite64(limit, mmio + limit_reg);
898 reg_val = ioread64(mmio + limit_reg);
899 if (reg_val != limit) {
900 iowrite64(base, mmio + limit_reg);
901 iowrite64(0, mmio + xlat_reg);
902 return -EIO;
903 }
904 } else {
905
906 if (addr & (~0ull << 32))
907 return -EINVAL;
908 if ((addr + size) & (~0ull << 32))
909 return -EINVAL;
910
911 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
912
913
914 if (limit_reg && size != mw_size)
915 limit = base + size;
916 else
917 limit = 0;
918
919
920 iowrite32(addr, mmio + xlat_reg);
921 reg_val = ioread32(mmio + xlat_reg);
922 if (reg_val != addr) {
923 iowrite32(0, mmio + xlat_reg);
924 return -EIO;
925 }
926
927
928 iowrite32(limit, mmio + limit_reg);
929 reg_val = ioread32(mmio + limit_reg);
930 if (reg_val != limit) {
931 iowrite32(base, mmio + limit_reg);
932 iowrite32(0, mmio + xlat_reg);
933 return -EIO;
934 }
935 }
936
937 return 0;
938}
939
940u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed,
941 enum ntb_width *width)
942{
943 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
944
945 if (ndev->reg->link_is_up(ndev)) {
946 if (speed)
947 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
948 if (width)
949 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
950 return 1;
951 } else {
952
953
954 if (speed)
955 *speed = NTB_SPEED_NONE;
956 if (width)
957 *width = NTB_WIDTH_NONE;
958 return 0;
959 }
960}
961
962static int intel_ntb_link_enable(struct ntb_dev *ntb,
963 enum ntb_speed max_speed,
964 enum ntb_width max_width)
965{
966 struct intel_ntb_dev *ndev;
967 u32 ntb_ctl;
968
969 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
970
971 if (ndev->ntb.topo == NTB_TOPO_SEC)
972 return -EINVAL;
973
974 dev_dbg(&ntb->pdev->dev,
975 "Enabling link with max_speed %d max_width %d\n",
976 max_speed, max_width);
977 if (max_speed != NTB_SPEED_AUTO)
978 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
979 if (max_width != NTB_WIDTH_AUTO)
980 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
981
982 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
983 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
984 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
985 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
986 if (ndev->bar4_split)
987 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
988 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
989
990 return 0;
991}
992
993int intel_ntb_link_disable(struct ntb_dev *ntb)
994{
995 struct intel_ntb_dev *ndev;
996 u32 ntb_cntl;
997
998 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
999
1000 if (ndev->ntb.topo == NTB_TOPO_SEC)
1001 return -EINVAL;
1002
1003 dev_dbg(&ntb->pdev->dev, "Disabling link\n");
1004
1005
1006 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1007 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1008 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1009 if (ndev->bar4_split)
1010 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1011 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1012 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1013
1014 return 0;
1015}
1016
1017int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
1018{
1019
1020 return ntb_ndev(ntb)->mw_count;
1021}
1022
1023int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
1024 phys_addr_t *base, resource_size_t *size)
1025{
1026 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1027 int bar;
1028
1029 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1030 idx += 1;
1031
1032 bar = ndev_mw_to_bar(ndev, idx);
1033 if (bar < 0)
1034 return bar;
1035
1036 if (base)
1037 *base = pci_resource_start(ndev->ntb.pdev, bar) +
1038 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1039
1040 if (size)
1041 *size = pci_resource_len(ndev->ntb.pdev, bar) -
1042 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1043
1044 return 0;
1045}
1046
1047static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1048{
1049 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1050}
1051
1052u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1053{
1054 return ntb_ndev(ntb)->db_valid_mask;
1055}
1056
1057int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1058{
1059 struct intel_ntb_dev *ndev;
1060
1061 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1062
1063 return ndev->db_vec_count;
1064}
1065
1066u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1067{
1068 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1069
1070 if (db_vector < 0 || db_vector > ndev->db_vec_count)
1071 return 0;
1072
1073 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1074}
1075
1076static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1077{
1078 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1079
1080 return ndev_db_read(ndev,
1081 ndev->self_mmio +
1082 ndev->self_reg->db_bell);
1083}
1084
1085static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1086{
1087 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1088
1089 return ndev_db_write(ndev, db_bits,
1090 ndev->self_mmio +
1091 ndev->self_reg->db_bell);
1092}
1093
1094int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1095{
1096 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1097
1098 return ndev_db_set_mask(ndev, db_bits,
1099 ndev->self_mmio +
1100 ndev->self_reg->db_mask);
1101}
1102
1103int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1104{
1105 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1106
1107 return ndev_db_clear_mask(ndev, db_bits,
1108 ndev->self_mmio +
1109 ndev->self_reg->db_mask);
1110}
1111
1112static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr,
1113 resource_size_t *db_size, u64 *db_data, int db_bit)
1114{
1115 u64 db_bits;
1116 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1117
1118 if (unlikely(db_bit >= BITS_PER_LONG_LONG))
1119 return -EINVAL;
1120
1121 db_bits = BIT_ULL(db_bit);
1122
1123 if (unlikely(db_bits & ~ntb_ndev(ntb)->db_valid_mask))
1124 return -EINVAL;
1125
1126 ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1127 ndev->peer_reg->db_bell);
1128
1129 if (db_data)
1130 *db_data = db_bits;
1131
1132
1133 return 0;
1134}
1135
1136static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1137{
1138 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1139
1140 return ndev_db_write(ndev, db_bits,
1141 ndev->peer_mmio +
1142 ndev->peer_reg->db_bell);
1143}
1144
1145int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1146{
1147 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1148}
1149
1150int intel_ntb_spad_count(struct ntb_dev *ntb)
1151{
1152 struct intel_ntb_dev *ndev;
1153
1154 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1155
1156 return ndev->spad_count;
1157}
1158
1159u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1160{
1161 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1162
1163 return ndev_spad_read(ndev, idx,
1164 ndev->self_mmio +
1165 ndev->self_reg->spad);
1166}
1167
1168int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
1169{
1170 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1171
1172 return ndev_spad_write(ndev, idx, val,
1173 ndev->self_mmio +
1174 ndev->self_reg->spad);
1175}
1176
1177int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
1178 phys_addr_t *spad_addr)
1179{
1180 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1181
1182 return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
1183 ndev->peer_reg->spad);
1184}
1185
1186u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
1187{
1188 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1189
1190 return ndev_spad_read(ndev, sidx,
1191 ndev->peer_mmio +
1192 ndev->peer_reg->spad);
1193}
1194
1195int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx,
1196 u32 val)
1197{
1198 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1199
1200 return ndev_spad_write(ndev, sidx, val,
1201 ndev->peer_mmio +
1202 ndev->peer_reg->spad);
1203}
1204
1205static u64 xeon_db_ioread(void __iomem *mmio)
1206{
1207 return (u64)ioread16(mmio);
1208}
1209
1210static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
1211{
1212 iowrite16((u16)bits, mmio);
1213}
1214
1215static int xeon_poll_link(struct intel_ntb_dev *ndev)
1216{
1217 u16 reg_val;
1218 int rc;
1219
1220 ndev->reg->db_iowrite(ndev->db_link_mask,
1221 ndev->self_mmio +
1222 ndev->self_reg->db_bell);
1223
1224 rc = pci_read_config_word(ndev->ntb.pdev,
1225 XEON_LINK_STATUS_OFFSET, ®_val);
1226 if (rc)
1227 return 0;
1228
1229 if (reg_val == ndev->lnk_sta)
1230 return 0;
1231
1232 ndev->lnk_sta = reg_val;
1233
1234 return 1;
1235}
1236
1237int xeon_link_is_up(struct intel_ntb_dev *ndev)
1238{
1239 if (ndev->ntb.topo == NTB_TOPO_SEC)
1240 return 1;
1241
1242 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
1243}
1244
1245enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
1246{
1247 switch (ppd & XEON_PPD_TOPO_MASK) {
1248 case XEON_PPD_TOPO_B2B_USD:
1249 return NTB_TOPO_B2B_USD;
1250
1251 case XEON_PPD_TOPO_B2B_DSD:
1252 return NTB_TOPO_B2B_DSD;
1253
1254 case XEON_PPD_TOPO_PRI_USD:
1255 case XEON_PPD_TOPO_PRI_DSD:
1256 return NTB_TOPO_PRI;
1257
1258 case XEON_PPD_TOPO_SEC_USD:
1259 case XEON_PPD_TOPO_SEC_DSD:
1260 return NTB_TOPO_SEC;
1261 }
1262
1263 return NTB_TOPO_NONE;
1264}
1265
1266static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
1267{
1268 if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
1269 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
1270 return 1;
1271 }
1272 return 0;
1273}
1274
1275static int xeon_init_isr(struct intel_ntb_dev *ndev)
1276{
1277 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
1278 XEON_DB_MSIX_VECTOR_COUNT,
1279 XEON_DB_MSIX_VECTOR_SHIFT,
1280 XEON_DB_TOTAL_SHIFT);
1281}
1282
1283static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
1284{
1285 ndev_deinit_isr(ndev);
1286}
1287
1288static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
1289 const struct intel_b2b_addr *addr,
1290 const struct intel_b2b_addr *peer_addr)
1291{
1292 struct pci_dev *pdev;
1293 void __iomem *mmio;
1294 resource_size_t bar_size;
1295 phys_addr_t bar_addr;
1296 int b2b_bar;
1297 u8 bar_sz;
1298
1299 pdev = ndev->ntb.pdev;
1300 mmio = ndev->self_mmio;
1301
1302 if (ndev->b2b_idx == UINT_MAX) {
1303 dev_dbg(&pdev->dev, "not using b2b mw\n");
1304 b2b_bar = 0;
1305 ndev->b2b_off = 0;
1306 } else {
1307 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1308 if (b2b_bar < 0)
1309 return -EIO;
1310
1311 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
1312
1313 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1314
1315 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
1316
1317 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
1318 dev_dbg(&pdev->dev, "b2b using first half of bar\n");
1319 ndev->b2b_off = bar_size >> 1;
1320 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
1321 dev_dbg(&pdev->dev, "b2b using whole bar\n");
1322 ndev->b2b_off = 0;
1323 --ndev->mw_count;
1324 } else {
1325 dev_dbg(&pdev->dev, "b2b bar size is too small\n");
1326 return -EIO;
1327 }
1328 }
1329
1330
1331
1332
1333
1334
1335
1336 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
1337 dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
1338 if (b2b_bar == 2) {
1339 if (ndev->b2b_off)
1340 bar_sz -= 1;
1341 else
1342 bar_sz = 0;
1343 }
1344 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
1345 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
1346 dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
1347
1348 if (!ndev->bar4_split) {
1349 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
1350 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
1351 if (b2b_bar == 4) {
1352 if (ndev->b2b_off)
1353 bar_sz -= 1;
1354 else
1355 bar_sz = 0;
1356 }
1357 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
1358 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
1359 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
1360 } else {
1361 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
1362 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
1363 if (b2b_bar == 4) {
1364 if (ndev->b2b_off)
1365 bar_sz -= 1;
1366 else
1367 bar_sz = 0;
1368 }
1369 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
1370 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
1371 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
1372
1373 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
1374 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
1375 if (b2b_bar == 5) {
1376 if (ndev->b2b_off)
1377 bar_sz -= 1;
1378 else
1379 bar_sz = 0;
1380 }
1381 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
1382 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
1383 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
1384 }
1385
1386
1387 if (b2b_bar == 0)
1388 bar_addr = addr->bar0_addr;
1389 else if (b2b_bar == 2)
1390 bar_addr = addr->bar2_addr64;
1391 else if (b2b_bar == 4 && !ndev->bar4_split)
1392 bar_addr = addr->bar4_addr64;
1393 else if (b2b_bar == 4)
1394 bar_addr = addr->bar4_addr32;
1395 else if (b2b_bar == 5)
1396 bar_addr = addr->bar5_addr32;
1397 else
1398 return -EIO;
1399
1400 dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
1401 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
1402
1403
1404
1405
1406
1407
1408 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1409 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
1410 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
1411 dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
1412
1413 if (!ndev->bar4_split) {
1414 bar_addr = addr->bar4_addr64 +
1415 (b2b_bar == 4 ? ndev->b2b_off : 0);
1416 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
1417 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
1418 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
1419 } else {
1420 bar_addr = addr->bar4_addr32 +
1421 (b2b_bar == 4 ? ndev->b2b_off : 0);
1422 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
1423 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
1424 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
1425
1426 bar_addr = addr->bar5_addr32 +
1427 (b2b_bar == 5 ? ndev->b2b_off : 0);
1428 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
1429 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
1430 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
1431 }
1432
1433
1434
1435 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1436 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
1437 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
1438 dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
1439
1440 if (!ndev->bar4_split) {
1441 bar_addr = addr->bar4_addr64 +
1442 (b2b_bar == 4 ? ndev->b2b_off : 0);
1443 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
1444 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
1445 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
1446 } else {
1447 bar_addr = addr->bar4_addr32 +
1448 (b2b_bar == 4 ? ndev->b2b_off : 0);
1449 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
1450 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
1451 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
1452
1453 bar_addr = addr->bar5_addr32 +
1454 (b2b_bar == 5 ? ndev->b2b_off : 0);
1455 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
1456 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
1457 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
1458 }
1459
1460
1461 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
1462
1463 if (!ndev->bar4_split) {
1464 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
1465 } else {
1466 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
1467 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
1468 }
1469
1470
1471 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
1472 if (!ndev->bar4_split) {
1473 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
1474 } else {
1475 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
1476 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
1477 }
1478
1479
1480 bar_addr = peer_addr->bar2_addr64;
1481 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
1482 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
1483 dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
1484
1485 if (!ndev->bar4_split) {
1486 bar_addr = peer_addr->bar4_addr64;
1487 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
1488 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
1489 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
1490 } else {
1491 bar_addr = peer_addr->bar4_addr32;
1492 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
1493 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
1494 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
1495
1496 bar_addr = peer_addr->bar5_addr32;
1497 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
1498 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
1499 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
1500 }
1501
1502
1503 if (b2b_bar == 0)
1504 bar_addr = peer_addr->bar0_addr;
1505 else if (b2b_bar == 2)
1506 bar_addr = peer_addr->bar2_addr64;
1507 else if (b2b_bar == 4 && !ndev->bar4_split)
1508 bar_addr = peer_addr->bar4_addr64;
1509 else if (b2b_bar == 4)
1510 bar_addr = peer_addr->bar4_addr32;
1511 else if (b2b_bar == 5)
1512 bar_addr = peer_addr->bar5_addr32;
1513 else
1514 return -EIO;
1515
1516
1517 dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
1518 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
1519 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
1520
1521 if (b2b_bar) {
1522
1523 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
1524 XEON_B2B_MIN_SIZE);
1525 if (!ndev->peer_mmio)
1526 return -EIO;
1527
1528 ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
1529 }
1530
1531 return 0;
1532}
1533
1534static int xeon_init_ntb(struct intel_ntb_dev *ndev)
1535{
1536 struct device *dev = &ndev->ntb.pdev->dev;
1537 int rc;
1538 u32 ntb_ctl;
1539
1540 if (ndev->bar4_split)
1541 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
1542 else
1543 ndev->mw_count = XEON_MW_COUNT;
1544
1545 ndev->spad_count = XEON_SPAD_COUNT;
1546 ndev->db_count = XEON_DB_COUNT;
1547 ndev->db_link_mask = XEON_DB_LINK_BIT;
1548
1549 switch (ndev->ntb.topo) {
1550 case NTB_TOPO_PRI:
1551 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1552 dev_err(dev, "NTB Primary config disabled\n");
1553 return -EINVAL;
1554 }
1555
1556
1557 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1558 ntb_ctl &= ~NTB_CTL_DISABLE;
1559 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1560
1561
1562 ndev->spad_count >>= 1;
1563 ndev->self_reg = &xeon_pri_reg;
1564 ndev->peer_reg = &xeon_sec_reg;
1565 ndev->xlat_reg = &xeon_sec_xlat;
1566 break;
1567
1568 case NTB_TOPO_SEC:
1569 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1570 dev_err(dev, "NTB Secondary config disabled\n");
1571 return -EINVAL;
1572 }
1573
1574 ndev->spad_count >>= 1;
1575 ndev->self_reg = &xeon_sec_reg;
1576 ndev->peer_reg = &xeon_pri_reg;
1577 ndev->xlat_reg = &xeon_pri_xlat;
1578 break;
1579
1580 case NTB_TOPO_B2B_USD:
1581 case NTB_TOPO_B2B_DSD:
1582 ndev->self_reg = &xeon_pri_reg;
1583 ndev->peer_reg = &xeon_b2b_reg;
1584 ndev->xlat_reg = &xeon_sec_xlat;
1585
1586 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
1587 ndev->peer_reg = &xeon_pri_reg;
1588
1589 if (b2b_mw_idx < 0)
1590 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
1591 else
1592 ndev->b2b_idx = b2b_mw_idx;
1593
1594 if (ndev->b2b_idx >= ndev->mw_count) {
1595 dev_dbg(dev,
1596 "b2b_mw_idx %d invalid for mw_count %u\n",
1597 b2b_mw_idx, ndev->mw_count);
1598 return -EINVAL;
1599 }
1600
1601 dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
1602 b2b_mw_idx, ndev->b2b_idx);
1603
1604 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
1605 dev_warn(dev, "Reduce doorbell count by 1\n");
1606 ndev->db_count -= 1;
1607 }
1608
1609 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1610 rc = xeon_setup_b2b_mw(ndev,
1611 &xeon_b2b_dsd_addr,
1612 &xeon_b2b_usd_addr);
1613 } else {
1614 rc = xeon_setup_b2b_mw(ndev,
1615 &xeon_b2b_usd_addr,
1616 &xeon_b2b_dsd_addr);
1617 }
1618 if (rc)
1619 return rc;
1620
1621
1622 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1623 ndev->self_mmio + XEON_SPCICMD_OFFSET);
1624
1625 break;
1626
1627 default:
1628 return -EINVAL;
1629 }
1630
1631 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1632
1633 ndev->reg->db_iowrite(ndev->db_valid_mask,
1634 ndev->self_mmio +
1635 ndev->self_reg->db_mask);
1636
1637 return 0;
1638}
1639
1640static int xeon_init_dev(struct intel_ntb_dev *ndev)
1641{
1642 struct pci_dev *pdev;
1643 u8 ppd;
1644 int rc, mem;
1645
1646 pdev = ndev->ntb.pdev;
1647
1648 switch (pdev->device) {
1649
1650
1651
1652
1653
1654
1655 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1656 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1657 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1658 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1659 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1660 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1661 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1662 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1663 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1664 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1665 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1666 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1667 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1668 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1669 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1670 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
1671 break;
1672 }
1673
1674 switch (pdev->device) {
1675
1676
1677
1678 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1679 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1680 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1681 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1682 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1683 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1684 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1685 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1686 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1687 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
1688 break;
1689 }
1690
1691 switch (pdev->device) {
1692
1693
1694
1695
1696 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
1697 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
1698 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
1699 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
1700 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
1701 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
1702 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
1703 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
1704 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
1705 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
1706 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
1707 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
1708 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
1709 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
1710 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
1711 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
1712 break;
1713 }
1714
1715 ndev->reg = &xeon_reg;
1716
1717 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1718 if (rc)
1719 return -EIO;
1720
1721 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1722 dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
1723 ntb_topo_string(ndev->ntb.topo));
1724 if (ndev->ntb.topo == NTB_TOPO_NONE)
1725 return -EINVAL;
1726
1727 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1728 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
1729 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
1730 ppd, ndev->bar4_split);
1731 } else {
1732
1733
1734
1735
1736 mem = pci_select_bars(pdev, IORESOURCE_MEM);
1737 ndev->bar4_split = hweight32(mem) ==
1738 HSX_SPLIT_BAR_MW_COUNT + 1;
1739 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
1740 mem, ndev->bar4_split);
1741 }
1742
1743 rc = xeon_init_ntb(ndev);
1744 if (rc)
1745 return rc;
1746
1747 return xeon_init_isr(ndev);
1748}
1749
1750static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
1751{
1752 xeon_deinit_isr(ndev);
1753}
1754
1755static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
1756{
1757 int rc;
1758
1759 pci_set_drvdata(pdev, ndev);
1760
1761 rc = pci_enable_device(pdev);
1762 if (rc)
1763 goto err_pci_enable;
1764
1765 rc = pci_request_regions(pdev, NTB_NAME);
1766 if (rc)
1767 goto err_pci_regions;
1768
1769 pci_set_master(pdev);
1770
1771 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
1772 if (rc) {
1773 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
1774 if (rc)
1775 goto err_dma_mask;
1776 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
1777 }
1778
1779 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
1780 if (rc) {
1781 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
1782 if (rc)
1783 goto err_dma_mask;
1784 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
1785 }
1786 rc = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
1787 dma_get_mask(&pdev->dev));
1788 if (rc)
1789 goto err_dma_mask;
1790
1791 ndev->self_mmio = pci_iomap(pdev, 0, 0);
1792 if (!ndev->self_mmio) {
1793 rc = -EIO;
1794 goto err_mmio;
1795 }
1796 ndev->peer_mmio = ndev->self_mmio;
1797 ndev->peer_addr = pci_resource_start(pdev, 0);
1798
1799 return 0;
1800
1801err_mmio:
1802err_dma_mask:
1803 pci_clear_master(pdev);
1804 pci_release_regions(pdev);
1805err_pci_regions:
1806 pci_disable_device(pdev);
1807err_pci_enable:
1808 pci_set_drvdata(pdev, NULL);
1809 return rc;
1810}
1811
1812static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
1813{
1814 struct pci_dev *pdev = ndev->ntb.pdev;
1815
1816 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
1817 pci_iounmap(pdev, ndev->peer_mmio);
1818 pci_iounmap(pdev, ndev->self_mmio);
1819
1820 pci_clear_master(pdev);
1821 pci_release_regions(pdev);
1822 pci_disable_device(pdev);
1823 pci_set_drvdata(pdev, NULL);
1824}
1825
1826static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
1827 struct pci_dev *pdev)
1828{
1829 ndev->ntb.pdev = pdev;
1830 ndev->ntb.topo = NTB_TOPO_NONE;
1831 ndev->ntb.ops = &intel_ntb_ops;
1832
1833 ndev->b2b_off = 0;
1834 ndev->b2b_idx = UINT_MAX;
1835
1836 ndev->bar4_split = 0;
1837
1838 ndev->mw_count = 0;
1839 ndev->spad_count = 0;
1840 ndev->db_count = 0;
1841 ndev->db_vec_count = 0;
1842 ndev->db_vec_shift = 0;
1843
1844 ndev->ntb_ctl = 0;
1845 ndev->lnk_sta = 0;
1846
1847 ndev->db_valid_mask = 0;
1848 ndev->db_link_mask = 0;
1849 ndev->db_mask = 0;
1850
1851 spin_lock_init(&ndev->db_mask_lock);
1852}
1853
1854static int intel_ntb_pci_probe(struct pci_dev *pdev,
1855 const struct pci_device_id *id)
1856{
1857 struct intel_ntb_dev *ndev;
1858 int rc, node;
1859
1860 node = dev_to_node(&pdev->dev);
1861
1862 if (pdev_is_gen1(pdev)) {
1863 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1864 if (!ndev) {
1865 rc = -ENOMEM;
1866 goto err_ndev;
1867 }
1868
1869 ndev_init_struct(ndev, pdev);
1870
1871 rc = intel_ntb_init_pci(ndev, pdev);
1872 if (rc)
1873 goto err_init_pci;
1874
1875 rc = xeon_init_dev(ndev);
1876 if (rc)
1877 goto err_init_dev;
1878
1879 } else if (pdev_is_gen3(pdev)) {
1880 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1881 if (!ndev) {
1882 rc = -ENOMEM;
1883 goto err_ndev;
1884 }
1885
1886 ndev_init_struct(ndev, pdev);
1887 ndev->ntb.ops = &intel_ntb3_ops;
1888
1889 rc = intel_ntb_init_pci(ndev, pdev);
1890 if (rc)
1891 goto err_init_pci;
1892
1893 rc = gen3_init_dev(ndev);
1894 if (rc)
1895 goto err_init_dev;
1896
1897 } else {
1898 rc = -EINVAL;
1899 goto err_ndev;
1900 }
1901
1902 ndev_reset_unsafe_flags(ndev);
1903
1904 ndev->reg->poll_link(ndev);
1905
1906 ndev_init_debugfs(ndev);
1907
1908 rc = ntb_register_device(&ndev->ntb);
1909 if (rc)
1910 goto err_register;
1911
1912 dev_info(&pdev->dev, "NTB device registered.\n");
1913
1914 return 0;
1915
1916err_register:
1917 ndev_deinit_debugfs(ndev);
1918 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
1919 xeon_deinit_dev(ndev);
1920err_init_dev:
1921 intel_ntb_deinit_pci(ndev);
1922err_init_pci:
1923 kfree(ndev);
1924err_ndev:
1925 return rc;
1926}
1927
1928static void intel_ntb_pci_remove(struct pci_dev *pdev)
1929{
1930 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
1931
1932 ntb_unregister_device(&ndev->ntb);
1933 ndev_deinit_debugfs(ndev);
1934 if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev))
1935 xeon_deinit_dev(ndev);
1936 intel_ntb_deinit_pci(ndev);
1937 kfree(ndev);
1938}
1939
1940static const struct intel_ntb_reg xeon_reg = {
1941 .poll_link = xeon_poll_link,
1942 .link_is_up = xeon_link_is_up,
1943 .db_ioread = xeon_db_ioread,
1944 .db_iowrite = xeon_db_iowrite,
1945 .db_size = sizeof(u32),
1946 .ntb_ctl = XEON_NTBCNTL_OFFSET,
1947 .mw_bar = {2, 4, 5},
1948};
1949
1950static const struct intel_ntb_alt_reg xeon_pri_reg = {
1951 .db_bell = XEON_PDOORBELL_OFFSET,
1952 .db_mask = XEON_PDBMSK_OFFSET,
1953 .spad = XEON_SPAD_OFFSET,
1954};
1955
1956static const struct intel_ntb_alt_reg xeon_sec_reg = {
1957 .db_bell = XEON_SDOORBELL_OFFSET,
1958 .db_mask = XEON_SDBMSK_OFFSET,
1959
1960 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
1961};
1962
1963static const struct intel_ntb_alt_reg xeon_b2b_reg = {
1964 .db_bell = XEON_B2B_DOORBELL_OFFSET,
1965 .spad = XEON_B2B_SPAD_OFFSET,
1966};
1967
1968static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979 .bar2_limit = XEON_PBAR23LMT_OFFSET,
1980 .bar2_xlat = XEON_PBAR23XLAT_OFFSET,
1981};
1982
1983static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
1984 .bar0_base = XEON_SBAR0BASE_OFFSET,
1985 .bar2_limit = XEON_SBAR23LMT_OFFSET,
1986 .bar2_xlat = XEON_SBAR23XLAT_OFFSET,
1987};
1988
1989struct intel_b2b_addr xeon_b2b_usd_addr = {
1990 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
1991 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
1992 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
1993 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
1994};
1995
1996struct intel_b2b_addr xeon_b2b_dsd_addr = {
1997 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
1998 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
1999 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
2000 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
2001};
2002
2003
2004static const struct ntb_dev_ops intel_ntb_ops = {
2005 .mw_count = intel_ntb_mw_count,
2006 .mw_get_align = intel_ntb_mw_get_align,
2007 .mw_set_trans = intel_ntb_mw_set_trans,
2008 .peer_mw_count = intel_ntb_peer_mw_count,
2009 .peer_mw_get_addr = intel_ntb_peer_mw_get_addr,
2010 .link_is_up = intel_ntb_link_is_up,
2011 .link_enable = intel_ntb_link_enable,
2012 .link_disable = intel_ntb_link_disable,
2013 .db_is_unsafe = intel_ntb_db_is_unsafe,
2014 .db_valid_mask = intel_ntb_db_valid_mask,
2015 .db_vector_count = intel_ntb_db_vector_count,
2016 .db_vector_mask = intel_ntb_db_vector_mask,
2017 .db_read = intel_ntb_db_read,
2018 .db_clear = intel_ntb_db_clear,
2019 .db_set_mask = intel_ntb_db_set_mask,
2020 .db_clear_mask = intel_ntb_db_clear_mask,
2021 .peer_db_addr = intel_ntb_peer_db_addr,
2022 .peer_db_set = intel_ntb_peer_db_set,
2023 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2024 .spad_count = intel_ntb_spad_count,
2025 .spad_read = intel_ntb_spad_read,
2026 .spad_write = intel_ntb_spad_write,
2027 .peer_spad_addr = intel_ntb_peer_spad_addr,
2028 .peer_spad_read = intel_ntb_peer_spad_read,
2029 .peer_spad_write = intel_ntb_peer_spad_write,
2030};
2031
2032static const struct file_operations intel_ntb_debugfs_info = {
2033 .owner = THIS_MODULE,
2034 .open = simple_open,
2035 .read = ndev_debugfs_read,
2036};
2037
2038static const struct pci_device_id intel_ntb_pci_tbl[] = {
2039 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2040 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2041 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2042 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2043 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
2044 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2045 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2046 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2047 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2048 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
2049 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2050 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2051 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2052 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2053 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
2054 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
2055 {0}
2056};
2057MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2058
2059static struct pci_driver intel_ntb_pci_driver = {
2060 .name = KBUILD_MODNAME,
2061 .id_table = intel_ntb_pci_tbl,
2062 .probe = intel_ntb_pci_probe,
2063 .remove = intel_ntb_pci_remove,
2064};
2065
2066static int __init intel_ntb_pci_driver_init(void)
2067{
2068 pr_info("%s %s\n", NTB_DESC, NTB_VER);
2069
2070 if (debugfs_initialized())
2071 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2072
2073 return pci_register_driver(&intel_ntb_pci_driver);
2074}
2075module_init(intel_ntb_pci_driver_init);
2076
2077static void __exit intel_ntb_pci_driver_exit(void)
2078{
2079 pci_unregister_driver(&intel_ntb_pci_driver);
2080
2081 debugfs_remove_recursive(debugfs_dir);
2082}
2083module_exit(intel_ntb_pci_driver_exit);
2084
