1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49#include <linux/debugfs.h>
50#include <linux/delay.h>
51#include <linux/init.h>
52#include <linux/interrupt.h>
53#include <linux/module.h>
54#include <linux/acpi.h>
55#include <linux/pci.h>
56#include <linux/random.h>
57#include <linux/slab.h>
58#include <linux/ntb.h>
59#include <linux/sizes.h>
60
61#include "ntb_hw_amd.h"
62
63#define NTB_NAME "ntb_hw_amd"
64#define NTB_DESC "AMD(R) PCI-E Non-Transparent Bridge Driver"
65#define NTB_VER "1.0"
66
67MODULE_DESCRIPTION(NTB_DESC);
68MODULE_VERSION(NTB_VER);
69MODULE_LICENSE("Dual BSD/GPL");
70MODULE_AUTHOR("AMD Inc.");
71
72static const struct file_operations amd_ntb_debugfs_info;
73static struct dentry *debugfs_dir;
74
75static int ndev_mw_to_bar(struct amd_ntb_dev *ndev, int idx)
76{
77 if (idx < 0 || idx > ndev->mw_count)
78 return -EINVAL;
79
80 return 1 << idx;
81}
82
83static int amd_ntb_mw_count(struct ntb_dev *ntb)
84{
85 return ntb_ndev(ntb)->mw_count;
86}
87
88static int amd_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
89 phys_addr_t *base,
90 resource_size_t *size,
91 resource_size_t *align,
92 resource_size_t *align_size)
93{
94 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
95 int bar;
96
97 bar = ndev_mw_to_bar(ndev, idx);
98 if (bar < 0)
99 return bar;
100
101 if (base)
102 *base = pci_resource_start(ndev->ntb.pdev, bar);
103
104 if (size)
105 *size = pci_resource_len(ndev->ntb.pdev, bar);
106
107 if (align)
108 *align = SZ_4K;
109
110 if (align_size)
111 *align_size = 1;
112
113 return 0;
114}
115
116static int amd_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
117 dma_addr_t addr, resource_size_t size)
118{
119 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
120 unsigned long xlat_reg, limit_reg = 0;
121 resource_size_t mw_size;
122 void __iomem *mmio, *peer_mmio;
123 u64 base_addr, limit, reg_val;
124 int bar;
125
126 bar = ndev_mw_to_bar(ndev, idx);
127 if (bar < 0)
128 return bar;
129
130 mw_size = pci_resource_len(ndev->ntb.pdev, bar);
131
132
133 if (size > mw_size)
134 return -EINVAL;
135
136 mmio = ndev->self_mmio;
137 peer_mmio = ndev->peer_mmio;
138
139 base_addr = pci_resource_start(ndev->ntb.pdev, bar);
140
141 if (bar != 1) {
142 xlat_reg = AMD_BAR23XLAT_OFFSET + ((bar - 2) << 2);
143 limit_reg = AMD_BAR23LMT_OFFSET + ((bar - 2) << 2);
144
145
146 limit = size;
147
148
149 write64(addr, peer_mmio + xlat_reg);
150 reg_val = read64(peer_mmio + xlat_reg);
151 if (reg_val != addr) {
152 write64(0, peer_mmio + xlat_reg);
153 return -EIO;
154 }
155
156
157 write64(limit, mmio + limit_reg);
158 reg_val = read64(mmio + limit_reg);
159 if (reg_val != limit) {
160 write64(base_addr, mmio + limit_reg);
161 write64(0, peer_mmio + xlat_reg);
162 return -EIO;
163 }
164 } else {
165 xlat_reg = AMD_BAR1XLAT_OFFSET;
166 limit_reg = AMD_BAR1LMT_OFFSET;
167
168
169 limit = size;
170
171
172 write64(addr, peer_mmio + xlat_reg);
173 reg_val = read64(peer_mmio + xlat_reg);
174 if (reg_val != addr) {
175 write64(0, peer_mmio + xlat_reg);
176 return -EIO;
177 }
178
179
180 writel(limit, mmio + limit_reg);
181 reg_val = readl(mmio + limit_reg);
182 if (reg_val != limit) {
183 writel(base_addr, mmio + limit_reg);
184 writel(0, peer_mmio + xlat_reg);
185 return -EIO;
186 }
187 }
188
189 return 0;
190}
191
192static int amd_link_is_up(struct amd_ntb_dev *ndev)
193{
194 if (!ndev->peer_sta)
195 return NTB_LNK_STA_ACTIVE(ndev->cntl_sta);
196
197 if (ndev->peer_sta & AMD_LINK_UP_EVENT) {
198 ndev->peer_sta = 0;
199 return 1;
200 }
201
202
203
204
205
206
207
208 if (ndev->peer_sta & AMD_PEER_RESET_EVENT)
209 ndev->peer_sta &= ~AMD_PEER_RESET_EVENT;
210 else if (ndev->peer_sta & (AMD_PEER_D0_EVENT | AMD_LINK_DOWN_EVENT))
211 ndev->peer_sta = 0;
212
213 return 0;
214}
215
216static int amd_ntb_link_is_up(struct ntb_dev *ntb,
217 enum ntb_speed *speed,
218 enum ntb_width *width)
219{
220 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
221 int ret = 0;
222
223 if (amd_link_is_up(ndev)) {
224 if (speed)
225 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
226 if (width)
227 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
228
229 dev_dbg(ndev_dev(ndev), "link is up.\n");
230
231 ret = 1;
232 } else {
233 if (speed)
234 *speed = NTB_SPEED_NONE;
235 if (width)
236 *width = NTB_WIDTH_NONE;
237
238 dev_dbg(ndev_dev(ndev), "link is down.\n");
239 }
240
241 return ret;
242}
243
244static int amd_ntb_link_enable(struct ntb_dev *ntb,
245 enum ntb_speed max_speed,
246 enum ntb_width max_width)
247{
248 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
249 void __iomem *mmio = ndev->self_mmio;
250 u32 ntb_ctl;
251
252
253 ndev->int_mask &= ~AMD_EVENT_INTMASK;
254 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
255
256 if (ndev->ntb.topo == NTB_TOPO_SEC)
257 return -EINVAL;
258 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
259
260 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
261 ntb_ctl |= (PMM_REG_CTL | SMM_REG_CTL);
262 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
263
264 return 0;
265}
266
267static int amd_ntb_link_disable(struct ntb_dev *ntb)
268{
269 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
270 void __iomem *mmio = ndev->self_mmio;
271 u32 ntb_ctl;
272
273
274 ndev->int_mask |= AMD_EVENT_INTMASK;
275 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
276
277 if (ndev->ntb.topo == NTB_TOPO_SEC)
278 return -EINVAL;
279 dev_dbg(ndev_dev(ndev), "Enabling Link.\n");
280
281 ntb_ctl = readl(mmio + AMD_CNTL_OFFSET);
282 ntb_ctl &= ~(PMM_REG_CTL | SMM_REG_CTL);
283 writel(ntb_ctl, mmio + AMD_CNTL_OFFSET);
284
285 return 0;
286}
287
288static u64 amd_ntb_db_valid_mask(struct ntb_dev *ntb)
289{
290 return ntb_ndev(ntb)->db_valid_mask;
291}
292
293static int amd_ntb_db_vector_count(struct ntb_dev *ntb)
294{
295 return ntb_ndev(ntb)->db_count;
296}
297
298static u64 amd_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
299{
300 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
301
302 if (db_vector < 0 || db_vector > ndev->db_count)
303 return 0;
304
305 return ntb_ndev(ntb)->db_valid_mask & (1 << db_vector);
306}
307
308static u64 amd_ntb_db_read(struct ntb_dev *ntb)
309{
310 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
311 void __iomem *mmio = ndev->self_mmio;
312
313 return (u64)readw(mmio + AMD_DBSTAT_OFFSET);
314}
315
316static int amd_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
317{
318 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
319 void __iomem *mmio = ndev->self_mmio;
320
321 writew((u16)db_bits, mmio + AMD_DBSTAT_OFFSET);
322
323 return 0;
324}
325
326static int amd_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
327{
328 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
329 void __iomem *mmio = ndev->self_mmio;
330 unsigned long flags;
331
332 if (db_bits & ~ndev->db_valid_mask)
333 return -EINVAL;
334
335 spin_lock_irqsave(&ndev->db_mask_lock, flags);
336 ndev->db_mask |= db_bits;
337 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
338 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
339
340 return 0;
341}
342
343static int amd_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
344{
345 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
346 void __iomem *mmio = ndev->self_mmio;
347 unsigned long flags;
348
349 if (db_bits & ~ndev->db_valid_mask)
350 return -EINVAL;
351
352 spin_lock_irqsave(&ndev->db_mask_lock, flags);
353 ndev->db_mask &= ~db_bits;
354 writew((u16)ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
355 spin_unlock_irqrestore(&ndev->db_mask_lock, flags);
356
357 return 0;
358}
359
360static int amd_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
361{
362 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
363 void __iomem *mmio = ndev->self_mmio;
364
365 writew((u16)db_bits, mmio + AMD_DBREQ_OFFSET);
366
367 return 0;
368}
369
370static int amd_ntb_spad_count(struct ntb_dev *ntb)
371{
372 return ntb_ndev(ntb)->spad_count;
373}
374
375static u32 amd_ntb_spad_read(struct ntb_dev *ntb, int idx)
376{
377 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
378 void __iomem *mmio = ndev->self_mmio;
379 u32 offset;
380
381 if (idx < 0 || idx >= ndev->spad_count)
382 return 0;
383
384 offset = ndev->self_spad + (idx << 2);
385 return readl(mmio + AMD_SPAD_OFFSET + offset);
386}
387
388static int amd_ntb_spad_write(struct ntb_dev *ntb,
389 int idx, u32 val)
390{
391 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
392 void __iomem *mmio = ndev->self_mmio;
393 u32 offset;
394
395 if (idx < 0 || idx >= ndev->spad_count)
396 return -EINVAL;
397
398 offset = ndev->self_spad + (idx << 2);
399 writel(val, mmio + AMD_SPAD_OFFSET + offset);
400
401 return 0;
402}
403
404static u32 amd_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
405{
406 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
407 void __iomem *mmio = ndev->self_mmio;
408 u32 offset;
409
410 if (idx < 0 || idx >= ndev->spad_count)
411 return -EINVAL;
412
413 offset = ndev->peer_spad + (idx << 2);
414 return readl(mmio + AMD_SPAD_OFFSET + offset);
415}
416
417static int amd_ntb_peer_spad_write(struct ntb_dev *ntb,
418 int idx, u32 val)
419{
420 struct amd_ntb_dev *ndev = ntb_ndev(ntb);
421 void __iomem *mmio = ndev->self_mmio;
422 u32 offset;
423
424 if (idx < 0 || idx >= ndev->spad_count)
425 return -EINVAL;
426
427 offset = ndev->peer_spad + (idx << 2);
428 writel(val, mmio + AMD_SPAD_OFFSET + offset);
429
430 return 0;
431}
432
433static const struct ntb_dev_ops amd_ntb_ops = {
434 .mw_count = amd_ntb_mw_count,
435 .mw_get_range = amd_ntb_mw_get_range,
436 .mw_set_trans = amd_ntb_mw_set_trans,
437 .link_is_up = amd_ntb_link_is_up,
438 .link_enable = amd_ntb_link_enable,
439 .link_disable = amd_ntb_link_disable,
440 .db_valid_mask = amd_ntb_db_valid_mask,
441 .db_vector_count = amd_ntb_db_vector_count,
442 .db_vector_mask = amd_ntb_db_vector_mask,
443 .db_read = amd_ntb_db_read,
444 .db_clear = amd_ntb_db_clear,
445 .db_set_mask = amd_ntb_db_set_mask,
446 .db_clear_mask = amd_ntb_db_clear_mask,
447 .peer_db_set = amd_ntb_peer_db_set,
448 .spad_count = amd_ntb_spad_count,
449 .spad_read = amd_ntb_spad_read,
450 .spad_write = amd_ntb_spad_write,
451 .peer_spad_read = amd_ntb_peer_spad_read,
452 .peer_spad_write = amd_ntb_peer_spad_write,
453};
454
455static void amd_ack_smu(struct amd_ntb_dev *ndev, u32 bit)
456{
457 void __iomem *mmio = ndev->self_mmio;
458 int reg;
459
460 reg = readl(mmio + AMD_SMUACK_OFFSET);
461 reg |= bit;
462 writel(reg, mmio + AMD_SMUACK_OFFSET);
463
464 ndev->peer_sta |= bit;
465}
466
467static void amd_handle_event(struct amd_ntb_dev *ndev, int vec)
468{
469 void __iomem *mmio = ndev->self_mmio;
470 u32 status;
471
472 status = readl(mmio + AMD_INTSTAT_OFFSET);
473 if (!(status & AMD_EVENT_INTMASK))
474 return;
475
476 dev_dbg(ndev_dev(ndev), "status = 0x%x and vec = %d\n", status, vec);
477
478 status &= AMD_EVENT_INTMASK;
479 switch (status) {
480 case AMD_PEER_FLUSH_EVENT:
481 dev_info(ndev_dev(ndev), "Flush is done.\n");
482 break;
483 case AMD_PEER_RESET_EVENT:
484 amd_ack_smu(ndev, AMD_PEER_RESET_EVENT);
485
486
487 ntb_link_event(&ndev->ntb);
488
489 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
490
491 break;
492 case AMD_PEER_D3_EVENT:
493 case AMD_PEER_PMETO_EVENT:
494 case AMD_LINK_UP_EVENT:
495 case AMD_LINK_DOWN_EVENT:
496 amd_ack_smu(ndev, status);
497
498
499 ntb_link_event(&ndev->ntb);
500
501 break;
502 case AMD_PEER_D0_EVENT:
503 mmio = ndev->peer_mmio;
504 status = readl(mmio + AMD_PMESTAT_OFFSET);
505
506 if (status & 0x1)
507 dev_info(ndev_dev(ndev), "Wakeup is done.\n");
508
509 amd_ack_smu(ndev, AMD_PEER_D0_EVENT);
510
511
512 schedule_delayed_work(&ndev->hb_timer,
513 AMD_LINK_HB_TIMEOUT);
514 break;
515 default:
516 dev_info(ndev_dev(ndev), "event status = 0x%x.\n", status);
517 break;
518 }
519}
520
521static irqreturn_t ndev_interrupt(struct amd_ntb_dev *ndev, int vec)
522{
523 dev_dbg(ndev_dev(ndev), "vec %d\n", vec);
524
525 if (vec > (AMD_DB_CNT - 1) || (ndev->msix_vec_count == 1))
526 amd_handle_event(ndev, vec);
527
528 if (vec < AMD_DB_CNT)
529 ntb_db_event(&ndev->ntb, vec);
530
531 return IRQ_HANDLED;
532}
533
534static irqreturn_t ndev_vec_isr(int irq, void *dev)
535{
536 struct amd_ntb_vec *nvec = dev;
537
538 return ndev_interrupt(nvec->ndev, nvec->num);
539}
540
541static irqreturn_t ndev_irq_isr(int irq, void *dev)
542{
543 struct amd_ntb_dev *ndev = dev;
544
545 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
546}
547
548static int ndev_init_isr(struct amd_ntb_dev *ndev,
549 int msix_min, int msix_max)
550{
551 struct pci_dev *pdev;
552 int rc, i, msix_count, node;
553
554 pdev = ndev_pdev(ndev);
555
556 node = dev_to_node(&pdev->dev);
557
558 ndev->db_mask = ndev->db_valid_mask;
559
560
561 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
562 GFP_KERNEL, node);
563 if (!ndev->vec)
564 goto err_msix_vec_alloc;
565
566 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
567 GFP_KERNEL, node);
568 if (!ndev->msix)
569 goto err_msix_alloc;
570
571 for (i = 0; i < msix_max; ++i)
572 ndev->msix[i].entry = i;
573
574 msix_count = pci_enable_msix_range(pdev, ndev->msix,
575 msix_min, msix_max);
576 if (msix_count < 0)
577 goto err_msix_enable;
578
579
580
581
582 if (msix_count < msix_min) {
583 pci_disable_msix(pdev);
584 goto err_msix_enable;
585 }
586
587 for (i = 0; i < msix_count; ++i) {
588 ndev->vec[i].ndev = ndev;
589 ndev->vec[i].num = i;
590 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
591 "ndev_vec_isr", &ndev->vec[i]);
592 if (rc)
593 goto err_msix_request;
594 }
595
596 dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
597 ndev->db_count = msix_min;
598 ndev->msix_vec_count = msix_max;
599 return 0;
600
601err_msix_request:
602 while (i-- > 0)
603 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
604 pci_disable_msix(pdev);
605err_msix_enable:
606 kfree(ndev->msix);
607err_msix_alloc:
608 kfree(ndev->vec);
609err_msix_vec_alloc:
610 ndev->msix = NULL;
611 ndev->vec = NULL;
612
613
614 rc = pci_enable_msi(pdev);
615 if (rc)
616 goto err_msi_enable;
617
618 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
619 "ndev_irq_isr", ndev);
620 if (rc)
621 goto err_msi_request;
622
623 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
624 ndev->db_count = 1;
625 ndev->msix_vec_count = 1;
626 return 0;
627
628err_msi_request:
629 pci_disable_msi(pdev);
630err_msi_enable:
631
632
633 pci_intx(pdev, 1);
634
635 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
636 "ndev_irq_isr", ndev);
637 if (rc)
638 goto err_intx_request;
639
640 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
641 ndev->db_count = 1;
642 ndev->msix_vec_count = 1;
643 return 0;
644
645err_intx_request:
646 return rc;
647}
648
649static void ndev_deinit_isr(struct amd_ntb_dev *ndev)
650{
651 struct pci_dev *pdev;
652 void __iomem *mmio = ndev->self_mmio;
653 int i;
654
655 pdev = ndev_pdev(ndev);
656
657
658 ndev->db_mask = ndev->db_valid_mask;
659 writel(ndev->db_mask, mmio + AMD_DBMASK_OFFSET);
660
661 if (ndev->msix) {
662 i = ndev->msix_vec_count;
663 while (i--)
664 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
665 pci_disable_msix(pdev);
666 kfree(ndev->msix);
667 kfree(ndev->vec);
668 } else {
669 free_irq(pdev->irq, ndev);
670 if (pci_dev_msi_enabled(pdev))
671 pci_disable_msi(pdev);
672 else
673 pci_intx(pdev, 0);
674 }
675}
676
677static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
678 size_t count, loff_t *offp)
679{
680 struct amd_ntb_dev *ndev;
681 void __iomem *mmio;
682 char *buf;
683 size_t buf_size;
684 ssize_t ret, off;
685 union { u64 v64; u32 v32; u16 v16; } u;
686
687 ndev = filp->private_data;
688 mmio = ndev->self_mmio;
689
690 buf_size = min(count, 0x800ul);
691
692 buf = kmalloc(buf_size, GFP_KERNEL);
693 if (!buf)
694 return -ENOMEM;
695
696 off = 0;
697
698 off += scnprintf(buf + off, buf_size - off,
699 "NTB Device Information:\n");
700
701 off += scnprintf(buf + off, buf_size - off,
702 "Connection Topology -\t%s\n",
703 ntb_topo_string(ndev->ntb.topo));
704
705 off += scnprintf(buf + off, buf_size - off,
706 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
707
708 if (!amd_link_is_up(ndev)) {
709 off += scnprintf(buf + off, buf_size - off,
710 "Link Status -\t\tDown\n");
711 } else {
712 off += scnprintf(buf + off, buf_size - off,
713 "Link Status -\t\tUp\n");
714 off += scnprintf(buf + off, buf_size - off,
715 "Link Speed -\t\tPCI-E Gen %u\n",
716 NTB_LNK_STA_SPEED(ndev->lnk_sta));
717 off += scnprintf(buf + off, buf_size - off,
718 "Link Width -\t\tx%u\n",
719 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
720 }
721
722 off += scnprintf(buf + off, buf_size - off,
723 "Memory Window Count -\t%u\n", ndev->mw_count);
724 off += scnprintf(buf + off, buf_size - off,
725 "Scratchpad Count -\t%u\n", ndev->spad_count);
726 off += scnprintf(buf + off, buf_size - off,
727 "Doorbell Count -\t%u\n", ndev->db_count);
728 off += scnprintf(buf + off, buf_size - off,
729 "MSIX Vector Count -\t%u\n", ndev->msix_vec_count);
730
731 off += scnprintf(buf + off, buf_size - off,
732 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
733
734 u.v32 = readl(ndev->self_mmio + AMD_DBMASK_OFFSET);
735 off += scnprintf(buf + off, buf_size - off,
736 "Doorbell Mask -\t\t\t%#06x\n", u.v32);
737
738 u.v32 = readl(mmio + AMD_DBSTAT_OFFSET);
739 off += scnprintf(buf + off, buf_size - off,
740 "Doorbell Bell -\t\t\t%#06x\n", u.v32);
741
742 off += scnprintf(buf + off, buf_size - off,
743 "\nNTB Incoming XLAT:\n");
744
745 u.v64 = read64(mmio + AMD_BAR1XLAT_OFFSET);
746 off += scnprintf(buf + off, buf_size - off,
747 "XLAT1 -\t\t%#018llx\n", u.v64);
748
749 u.v64 = read64(ndev->self_mmio + AMD_BAR23XLAT_OFFSET);
750 off += scnprintf(buf + off, buf_size - off,
751 "XLAT23 -\t\t%#018llx\n", u.v64);
752
753 u.v64 = read64(ndev->self_mmio + AMD_BAR45XLAT_OFFSET);
754 off += scnprintf(buf + off, buf_size - off,
755 "XLAT45 -\t\t%#018llx\n", u.v64);
756
757 u.v32 = readl(mmio + AMD_BAR1LMT_OFFSET);
758 off += scnprintf(buf + off, buf_size - off,
759 "LMT1 -\t\t\t%#06x\n", u.v32);
760
761 u.v64 = read64(ndev->self_mmio + AMD_BAR23LMT_OFFSET);
762 off += scnprintf(buf + off, buf_size - off,
763 "LMT23 -\t\t\t%#018llx\n", u.v64);
764
765 u.v64 = read64(ndev->self_mmio + AMD_BAR45LMT_OFFSET);
766 off += scnprintf(buf + off, buf_size - off,
767 "LMT45 -\t\t\t%#018llx\n", u.v64);
768
769 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
770 kfree(buf);
771 return ret;
772}
773
774static void ndev_init_debugfs(struct amd_ntb_dev *ndev)
775{
776 if (!debugfs_dir) {
777 ndev->debugfs_dir = NULL;
778 ndev->debugfs_info = NULL;
779 } else {
780 ndev->debugfs_dir =
781 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
782 if (!ndev->debugfs_dir)
783 ndev->debugfs_info = NULL;
784 else
785 ndev->debugfs_info =
786 debugfs_create_file("info", S_IRUSR,
787 ndev->debugfs_dir, ndev,
788 &amd_ntb_debugfs_info);
789 }
790}
791
792static void ndev_deinit_debugfs(struct amd_ntb_dev *ndev)
793{
794 debugfs_remove_recursive(ndev->debugfs_dir);
795}
796
797static inline void ndev_init_struct(struct amd_ntb_dev *ndev,
798 struct pci_dev *pdev)
799{
800 ndev->ntb.pdev = pdev;
801 ndev->ntb.topo = NTB_TOPO_NONE;
802 ndev->ntb.ops = &amd_ntb_ops;
803 ndev->int_mask = AMD_EVENT_INTMASK;
804 spin_lock_init(&ndev->db_mask_lock);
805}
806
807static int amd_poll_link(struct amd_ntb_dev *ndev)
808{
809 void __iomem *mmio = ndev->peer_mmio;
810 u32 reg, stat;
811 int rc;
812
813 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
814 reg &= NTB_LIN_STA_ACTIVE_BIT;
815
816 dev_dbg(ndev_dev(ndev), "%s: reg_val = 0x%x.\n", __func__, reg);
817
818 if (reg == ndev->cntl_sta)
819 return 0;
820
821 ndev->cntl_sta = reg;
822
823 rc = pci_read_config_dword(ndev->ntb.pdev,
824 AMD_LINK_STATUS_OFFSET, &stat);
825 if (rc)
826 return 0;
827 ndev->lnk_sta = stat;
828
829 return 1;
830}
831
832static void amd_link_hb(struct work_struct *work)
833{
834 struct amd_ntb_dev *ndev = hb_ndev(work);
835
836 if (amd_poll_link(ndev))
837 ntb_link_event(&ndev->ntb);
838
839 if (!amd_link_is_up(ndev))
840 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
841}
842
843static int amd_init_isr(struct amd_ntb_dev *ndev)
844{
845 return ndev_init_isr(ndev, AMD_DB_CNT, AMD_MSIX_VECTOR_CNT);
846}
847
848static void amd_init_side_info(struct amd_ntb_dev *ndev)
849{
850 void __iomem *mmio = ndev->self_mmio;
851 unsigned int reg;
852
853 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
854 if (!(reg & AMD_SIDE_READY)) {
855 reg |= AMD_SIDE_READY;
856 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
857 }
858}
859
860static void amd_deinit_side_info(struct amd_ntb_dev *ndev)
861{
862 void __iomem *mmio = ndev->self_mmio;
863 unsigned int reg;
864
865 reg = readl(mmio + AMD_SIDEINFO_OFFSET);
866 if (reg & AMD_SIDE_READY) {
867 reg &= ~AMD_SIDE_READY;
868 writel(reg, mmio + AMD_SIDEINFO_OFFSET);
869 readl(mmio + AMD_SIDEINFO_OFFSET);
870 }
871}
872
873static int amd_init_ntb(struct amd_ntb_dev *ndev)
874{
875 void __iomem *mmio = ndev->self_mmio;
876
877 ndev->mw_count = AMD_MW_CNT;
878 ndev->spad_count = AMD_SPADS_CNT;
879 ndev->db_count = AMD_DB_CNT;
880
881 switch (ndev->ntb.topo) {
882 case NTB_TOPO_PRI:
883 case NTB_TOPO_SEC:
884 ndev->spad_count >>= 1;
885 if (ndev->ntb.topo == NTB_TOPO_PRI) {
886 ndev->self_spad = 0;
887 ndev->peer_spad = 0x20;
888 } else {
889 ndev->self_spad = 0x20;
890 ndev->peer_spad = 0;
891 }
892
893 INIT_DELAYED_WORK(&ndev->hb_timer, amd_link_hb);
894 schedule_delayed_work(&ndev->hb_timer, AMD_LINK_HB_TIMEOUT);
895
896 break;
897 default:
898 dev_err(ndev_dev(ndev), "AMD NTB does not support B2B mode.\n");
899 return -EINVAL;
900 }
901
902 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
903
904
905 writel(ndev->int_mask, mmio + AMD_INTMASK_OFFSET);
906
907 return 0;
908}
909
910static enum ntb_topo amd_get_topo(struct amd_ntb_dev *ndev)
911{
912 void __iomem *mmio = ndev->self_mmio;
913 u32 info;
914
915 info = readl(mmio + AMD_SIDEINFO_OFFSET);
916 if (info & AMD_SIDE_MASK)
917 return NTB_TOPO_SEC;
918 else
919 return NTB_TOPO_PRI;
920}
921
922static int amd_init_dev(struct amd_ntb_dev *ndev)
923{
924 struct pci_dev *pdev;
925 int rc = 0;
926
927 pdev = ndev_pdev(ndev);
928
929 ndev->ntb.topo = amd_get_topo(ndev);
930 dev_dbg(ndev_dev(ndev), "AMD NTB topo is %s\n",
931 ntb_topo_string(ndev->ntb.topo));
932
933 rc = amd_init_ntb(ndev);
934 if (rc)
935 return rc;
936
937 rc = amd_init_isr(ndev);
938 if (rc) {
939 dev_err(ndev_dev(ndev), "fail to init isr.\n");
940 return rc;
941 }
942
943 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
944
945 return 0;
946}
947
948static void amd_deinit_dev(struct amd_ntb_dev *ndev)
949{
950 cancel_delayed_work_sync(&ndev->hb_timer);
951
952 ndev_deinit_isr(ndev);
953}
954
955static int amd_ntb_init_pci(struct amd_ntb_dev *ndev,
956 struct pci_dev *pdev)
957{
958 int rc;
959
960 pci_set_drvdata(pdev, ndev);
961
962 rc = pci_enable_device(pdev);
963 if (rc)
964 goto err_pci_enable;
965
966 rc = pci_request_regions(pdev, NTB_NAME);
967 if (rc)
968 goto err_pci_regions;
969
970 pci_set_master(pdev);
971
972 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
973 if (rc) {
974 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
975 if (rc)
976 goto err_dma_mask;
977 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
978 }
979
980 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
981 if (rc) {
982 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
983 if (rc)
984 goto err_dma_mask;
985 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
986 }
987
988 ndev->self_mmio = pci_iomap(pdev, 0, 0);
989 if (!ndev->self_mmio) {
990 rc = -EIO;
991 goto err_dma_mask;
992 }
993 ndev->peer_mmio = ndev->self_mmio + AMD_PEER_OFFSET;
994
995 return 0;
996
997err_dma_mask:
998 pci_clear_master(pdev);
999err_pci_regions:
1000 pci_disable_device(pdev);
1001err_pci_enable:
1002 pci_set_drvdata(pdev, NULL);
1003 return rc;
1004}
1005
1006static void amd_ntb_deinit_pci(struct amd_ntb_dev *ndev)
1007{
1008 struct pci_dev *pdev = ndev_pdev(ndev);
1009
1010 pci_iounmap(pdev, ndev->self_mmio);
1011
1012 pci_clear_master(pdev);
1013 pci_release_regions(pdev);
1014 pci_disable_device(pdev);
1015 pci_set_drvdata(pdev, NULL);
1016}
1017
1018static int amd_ntb_pci_probe(struct pci_dev *pdev,
1019 const struct pci_device_id *id)
1020{
1021 struct amd_ntb_dev *ndev;
1022 int rc, node;
1023
1024 node = dev_to_node(&pdev->dev);
1025
1026 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
1027 if (!ndev) {
1028 rc = -ENOMEM;
1029 goto err_ndev;
1030 }
1031
1032 ndev_init_struct(ndev, pdev);
1033
1034 rc = amd_ntb_init_pci(ndev, pdev);
1035 if (rc)
1036 goto err_init_pci;
1037
1038 rc = amd_init_dev(ndev);
1039 if (rc)
1040 goto err_init_dev;
1041
1042
1043 amd_init_side_info(ndev);
1044
1045 amd_poll_link(ndev);
1046
1047 ndev_init_debugfs(ndev);
1048
1049 rc = ntb_register_device(&ndev->ntb);
1050 if (rc)
1051 goto err_register;
1052
1053 dev_info(&pdev->dev, "NTB device registered.\n");
1054
1055 return 0;
1056
1057err_register:
1058 ndev_deinit_debugfs(ndev);
1059 amd_deinit_dev(ndev);
1060err_init_dev:
1061 amd_ntb_deinit_pci(ndev);
1062err_init_pci:
1063 kfree(ndev);
1064err_ndev:
1065 return rc;
1066}
1067
1068static void amd_ntb_pci_remove(struct pci_dev *pdev)
1069{
1070 struct amd_ntb_dev *ndev = pci_get_drvdata(pdev);
1071
1072 ntb_unregister_device(&ndev->ntb);
1073 ndev_deinit_debugfs(ndev);
1074 amd_deinit_side_info(ndev);
1075 amd_deinit_dev(ndev);
1076 amd_ntb_deinit_pci(ndev);
1077 kfree(ndev);
1078}
1079
1080static const struct file_operations amd_ntb_debugfs_info = {
1081 .owner = THIS_MODULE,
1082 .open = simple_open,
1083 .read = ndev_debugfs_read,
1084};
1085
1086static const struct pci_device_id amd_ntb_pci_tbl[] = {
1087 {PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_NTB)},
1088 {0}
1089};
1090MODULE_DEVICE_TABLE(pci, amd_ntb_pci_tbl);
1091
1092static struct pci_driver amd_ntb_pci_driver = {
1093 .name = KBUILD_MODNAME,
1094 .id_table = amd_ntb_pci_tbl,
1095 .probe = amd_ntb_pci_probe,
1096 .remove = amd_ntb_pci_remove,
1097};
1098
1099static int __init amd_ntb_pci_driver_init(void)
1100{
1101 pr_info("%s %s\n", NTB_DESC, NTB_VER);
1102
1103 if (debugfs_initialized())
1104 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1105
1106 return pci_register_driver(&amd_ntb_pci_driver);
1107}
1108module_init(amd_ntb_pci_driver_init);
1109
1110static void __exit amd_ntb_pci_driver_exit(void)
1111{
1112 pci_unregister_driver(&amd_ntb_pci_driver);
1113 debugfs_remove_recursive(debugfs_dir);
1114}
1115module_exit(amd_ntb_pci_driver_exit);
1116
