1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#include <linux/spinlock.h>
49#include <linux/pci.h>
50#include <linux/io.h>
51#include <linux/delay.h>
52#include <linux/netdevice.h>
53#include <linux/vmalloc.h>
54#include <linux/module.h>
55#include <linux/prefetch.h>
56#include <rdma/ib_verbs.h>
57
58#include "hfi.h"
59#include "trace.h"
60#include "qp.h"
61#include "sdma.h"
62#include "debugfs.h"
63#include "vnic.h"
64#include "fault.h"
65
66#undef pr_fmt
67#define pr_fmt(fmt) DRIVER_NAME ": " fmt
68
69
70
71
72
73const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";
74
75DEFINE_MUTEX(hfi1_mutex);
76
77unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
78module_param_named(max_mtu, hfi1_max_mtu, uint, S_IRUGO);
79MODULE_PARM_DESC(max_mtu, "Set max MTU bytes, default is " __stringify(
80 HFI1_DEFAULT_MAX_MTU));
81
82unsigned int hfi1_cu = 1;
83module_param_named(cu, hfi1_cu, uint, S_IRUGO);
84MODULE_PARM_DESC(cu, "Credit return units");
85
86unsigned long hfi1_cap_mask = HFI1_CAP_MASK_DEFAULT;
87static int hfi1_caps_set(const char *val, const struct kernel_param *kp);
88static int hfi1_caps_get(char *buffer, const struct kernel_param *kp);
89static const struct kernel_param_ops cap_ops = {
90 .set = hfi1_caps_set,
91 .get = hfi1_caps_get
92};
93module_param_cb(cap_mask, &cap_ops, &hfi1_cap_mask, S_IWUSR | S_IRUGO);
94MODULE_PARM_DESC(cap_mask, "Bit mask of enabled/disabled HW features");
95
96MODULE_LICENSE("Dual BSD/GPL");
97MODULE_DESCRIPTION("Intel Omni-Path Architecture driver");
98
99
100
101
102#define MAX_PKT_RECV 64
103
104
105
106
107#define MAX_PKT_RECV_THREAD (MAX_PKT_RECV * 4)
108#define EGR_HEAD_UPDATE_THRESHOLD 16
109
110struct hfi1_ib_stats hfi1_stats;
111
112static int hfi1_caps_set(const char *val, const struct kernel_param *kp)
113{
114 int ret = 0;
115 unsigned long *cap_mask_ptr = (unsigned long *)kp->arg,
116 cap_mask = *cap_mask_ptr, value, diff,
117 write_mask = ((HFI1_CAP_WRITABLE_MASK << HFI1_CAP_USER_SHIFT) |
118 HFI1_CAP_WRITABLE_MASK);
119
120 ret = kstrtoul(val, 0, &value);
121 if (ret) {
122 pr_warn("Invalid module parameter value for 'cap_mask'\n");
123 goto done;
124 }
125
126 diff = value ^ (cap_mask & ~HFI1_CAP_LOCKED_SMASK);
127
128
129 if (HFI1_CAP_LOCKED() && (diff & ~write_mask)) {
130 pr_warn("Ignoring non-writable capability bits %#lx\n",
131 diff & ~write_mask);
132 diff &= write_mask;
133 }
134
135
136 diff &= ~HFI1_CAP_RESERVED_MASK;
137
138 cap_mask &= ~diff;
139
140 cap_mask |= (value & diff);
141
142 diff = (cap_mask & (HFI1_CAP_MUST_HAVE_KERN << HFI1_CAP_USER_SHIFT)) ^
143 ((cap_mask & HFI1_CAP_MUST_HAVE_KERN) << HFI1_CAP_USER_SHIFT);
144 cap_mask &= ~diff;
145
146 *cap_mask_ptr = cap_mask;
147done:
148 return ret;
149}
150
151static int hfi1_caps_get(char *buffer, const struct kernel_param *kp)
152{
153 unsigned long cap_mask = *(unsigned long *)kp->arg;
154
155 cap_mask &= ~HFI1_CAP_LOCKED_SMASK;
156 cap_mask |= ((cap_mask & HFI1_CAP_K2U) << HFI1_CAP_USER_SHIFT);
157
158 return scnprintf(buffer, PAGE_SIZE, "0x%lx", cap_mask);
159}
160
161struct pci_dev *get_pci_dev(struct rvt_dev_info *rdi)
162{
163 struct hfi1_ibdev *ibdev = container_of(rdi, struct hfi1_ibdev, rdi);
164 struct hfi1_devdata *dd = container_of(ibdev,
165 struct hfi1_devdata, verbs_dev);
166 return dd->pcidev;
167}
168
169
170
171
172int hfi1_count_active_units(void)
173{
174 struct hfi1_devdata *dd;
175 struct hfi1_pportdata *ppd;
176 unsigned long index, flags;
177 int pidx, nunits_active = 0;
178
179 xa_lock_irqsave(&hfi1_dev_table, flags);
180 xa_for_each(&hfi1_dev_table, index, dd) {
181 if (!(dd->flags & HFI1_PRESENT) || !dd->kregbase1)
182 continue;
183 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
184 ppd = dd->pport + pidx;
185 if (ppd->lid && ppd->linkup) {
186 nunits_active++;
187 break;
188 }
189 }
190 }
191 xa_unlock_irqrestore(&hfi1_dev_table, flags);
192 return nunits_active;
193}
194
195
196
197
198
199static inline void *get_egrbuf(const struct hfi1_ctxtdata *rcd, u64 rhf,
200 u8 *update)
201{
202 u32 idx = rhf_egr_index(rhf), offset = rhf_egr_buf_offset(rhf);
203
204 *update |= !(idx & (rcd->egrbufs.threshold - 1)) && !offset;
205 return (void *)(((u64)(rcd->egrbufs.rcvtids[idx].addr)) +
206 (offset * RCV_BUF_BLOCK_SIZE));
207}
208
209static inline void *hfi1_get_header(struct hfi1_ctxtdata *rcd,
210 __le32 *rhf_addr)
211{
212 u32 offset = rhf_hdrq_offset(rhf_to_cpu(rhf_addr));
213
214 return (void *)(rhf_addr - rcd->rhf_offset + offset);
215}
216
217static inline struct ib_header *hfi1_get_msgheader(struct hfi1_ctxtdata *rcd,
218 __le32 *rhf_addr)
219{
220 return (struct ib_header *)hfi1_get_header(rcd, rhf_addr);
221}
222
223static inline struct hfi1_16b_header
224 *hfi1_get_16B_header(struct hfi1_ctxtdata *rcd,
225 __le32 *rhf_addr)
226{
227 return (struct hfi1_16b_header *)hfi1_get_header(rcd, rhf_addr);
228}
229
230
231
232
233
234
235
236int hfi1_rcvbuf_validate(u32 size, u8 type, u16 *encoded)
237{
238 if (unlikely(!PAGE_ALIGNED(size)))
239 return 0;
240 if (unlikely(size < MIN_EAGER_BUFFER))
241 return 0;
242 if (size >
243 (type == PT_EAGER ? MAX_EAGER_BUFFER : MAX_EXPECTED_BUFFER))
244 return 0;
245 if (encoded)
246 *encoded = ilog2(size / PAGE_SIZE) + 1;
247 return 1;
248}
249
250static void rcv_hdrerr(struct hfi1_ctxtdata *rcd, struct hfi1_pportdata *ppd,
251 struct hfi1_packet *packet)
252{
253 struct ib_header *rhdr = packet->hdr;
254 u32 rte = rhf_rcv_type_err(packet->rhf);
255 u32 mlid_base;
256 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
257 struct hfi1_devdata *dd = ppd->dd;
258 struct hfi1_ibdev *verbs_dev = &dd->verbs_dev;
259 struct rvt_dev_info *rdi = &verbs_dev->rdi;
260
261 if ((packet->rhf & RHF_DC_ERR) &&
262 hfi1_dbg_fault_suppress_err(verbs_dev))
263 return;
264
265 if (packet->rhf & RHF_ICRC_ERR)
266 return;
267
268 if (packet->etype == RHF_RCV_TYPE_BYPASS) {
269 goto drop;
270 } else {
271 u8 lnh = ib_get_lnh(rhdr);
272
273 mlid_base = be16_to_cpu(IB_MULTICAST_LID_BASE);
274 if (lnh == HFI1_LRH_BTH) {
275 packet->ohdr = &rhdr->u.oth;
276 } else if (lnh == HFI1_LRH_GRH) {
277 packet->ohdr = &rhdr->u.l.oth;
278 packet->grh = &rhdr->u.l.grh;
279 } else {
280 goto drop;
281 }
282 }
283
284 if (packet->rhf & RHF_TID_ERR) {
285
286 u32 tlen = rhf_pkt_len(packet->rhf);
287 u32 dlid = ib_get_dlid(rhdr);
288 u32 qp_num;
289
290
291 if (tlen < 24)
292 goto drop;
293
294
295 if (packet->grh) {
296 u32 vtf;
297 struct ib_grh *grh = packet->grh;
298
299 if (grh->next_hdr != IB_GRH_NEXT_HDR)
300 goto drop;
301 vtf = be32_to_cpu(grh->version_tclass_flow);
302 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
303 goto drop;
304 }
305
306
307 qp_num = ib_bth_get_qpn(packet->ohdr);
308 if (dlid < mlid_base) {
309 struct rvt_qp *qp;
310 unsigned long flags;
311
312 rcu_read_lock();
313 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qp_num);
314 if (!qp) {
315 rcu_read_unlock();
316 goto drop;
317 }
318
319
320
321
322
323 spin_lock_irqsave(&qp->r_lock, flags);
324
325
326 if (!(ib_rvt_state_ops[qp->state] &
327 RVT_PROCESS_RECV_OK)) {
328 ibp->rvp.n_pkt_drops++;
329 }
330
331 switch (qp->ibqp.qp_type) {
332 case IB_QPT_RC:
333 hfi1_rc_hdrerr(rcd, packet, qp);
334 break;
335 default:
336
337 break;
338 }
339
340 spin_unlock_irqrestore(&qp->r_lock, flags);
341 rcu_read_unlock();
342 }
343 }
344
345
346 switch (rte) {
347 case RHF_RTE_ERROR_OP_CODE_ERR:
348 {
349 void *ebuf = NULL;
350 u8 opcode;
351
352 if (rhf_use_egr_bfr(packet->rhf))
353 ebuf = packet->ebuf;
354
355 if (!ebuf)
356 goto drop;
357
358 opcode = ib_bth_get_opcode(packet->ohdr);
359 if (opcode == IB_OPCODE_CNP) {
360
361
362
363
364 struct rvt_qp *qp = NULL;
365 u32 lqpn, rqpn;
366 u16 rlid;
367 u8 svc_type, sl, sc5;
368
369 sc5 = hfi1_9B_get_sc5(rhdr, packet->rhf);
370 sl = ibp->sc_to_sl[sc5];
371
372 lqpn = ib_bth_get_qpn(packet->ohdr);
373 rcu_read_lock();
374 qp = rvt_lookup_qpn(rdi, &ibp->rvp, lqpn);
375 if (!qp) {
376 rcu_read_unlock();
377 goto drop;
378 }
379
380 switch (qp->ibqp.qp_type) {
381 case IB_QPT_UD:
382 rlid = 0;
383 rqpn = 0;
384 svc_type = IB_CC_SVCTYPE_UD;
385 break;
386 case IB_QPT_UC:
387 rlid = ib_get_slid(rhdr);
388 rqpn = qp->remote_qpn;
389 svc_type = IB_CC_SVCTYPE_UC;
390 break;
391 default:
392 rcu_read_unlock();
393 goto drop;
394 }
395
396 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
397 rcu_read_unlock();
398 }
399
400 packet->rhf &= ~RHF_RCV_TYPE_ERR_SMASK;
401 break;
402 }
403 default:
404 break;
405 }
406
407drop:
408 return;
409}
410
411static inline void init_packet(struct hfi1_ctxtdata *rcd,
412 struct hfi1_packet *packet)
413{
414 packet->rsize = rcd->rcvhdrqentsize;
415 packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize;
416 packet->rcd = rcd;
417 packet->updegr = 0;
418 packet->etail = -1;
419 packet->rhf_addr = get_rhf_addr(rcd);
420 packet->rhf = rhf_to_cpu(packet->rhf_addr);
421 packet->rhqoff = rcd->head;
422 packet->numpkt = 0;
423}
424
425
426static const hfi1_handle_cnp hfi1_handle_cnp_tbl[2] = {
427 [HFI1_PKT_TYPE_9B] = &return_cnp,
428 [HFI1_PKT_TYPE_16B] = &return_cnp_16B
429};
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444bool hfi1_process_ecn_slowpath(struct rvt_qp *qp, struct hfi1_packet *pkt,
445 bool prescan)
446{
447 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
448 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
449 struct ib_other_headers *ohdr = pkt->ohdr;
450 struct ib_grh *grh = pkt->grh;
451 u32 rqpn = 0;
452 u16 pkey;
453 u32 rlid, slid, dlid = 0;
454 u8 hdr_type, sc, svc_type, opcode;
455 bool is_mcast = false, ignore_fecn = false, do_cnp = false,
456 fecn, becn;
457
458
459 if (pkt->etype == RHF_RCV_TYPE_BYPASS) {
460 pkey = hfi1_16B_get_pkey(pkt->hdr);
461 sc = hfi1_16B_get_sc(pkt->hdr);
462 dlid = hfi1_16B_get_dlid(pkt->hdr);
463 slid = hfi1_16B_get_slid(pkt->hdr);
464 is_mcast = hfi1_is_16B_mcast(dlid);
465 opcode = ib_bth_get_opcode(ohdr);
466 hdr_type = HFI1_PKT_TYPE_16B;
467 fecn = hfi1_16B_get_fecn(pkt->hdr);
468 becn = hfi1_16B_get_becn(pkt->hdr);
469 } else {
470 pkey = ib_bth_get_pkey(ohdr);
471 sc = hfi1_9B_get_sc5(pkt->hdr, pkt->rhf);
472 dlid = qp->ibqp.qp_type != IB_QPT_UD ? ib_get_dlid(pkt->hdr) :
473 ppd->lid;
474 slid = ib_get_slid(pkt->hdr);
475 is_mcast = (dlid > be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
476 (dlid != be16_to_cpu(IB_LID_PERMISSIVE));
477 opcode = ib_bth_get_opcode(ohdr);
478 hdr_type = HFI1_PKT_TYPE_9B;
479 fecn = ib_bth_get_fecn(ohdr);
480 becn = ib_bth_get_becn(ohdr);
481 }
482
483 switch (qp->ibqp.qp_type) {
484 case IB_QPT_UD:
485 rlid = slid;
486 rqpn = ib_get_sqpn(pkt->ohdr);
487 svc_type = IB_CC_SVCTYPE_UD;
488 break;
489 case IB_QPT_SMI:
490 case IB_QPT_GSI:
491 rlid = slid;
492 rqpn = ib_get_sqpn(pkt->ohdr);
493 svc_type = IB_CC_SVCTYPE_UD;
494 break;
495 case IB_QPT_UC:
496 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
497 rqpn = qp->remote_qpn;
498 svc_type = IB_CC_SVCTYPE_UC;
499 break;
500 case IB_QPT_RC:
501 rlid = rdma_ah_get_dlid(&qp->remote_ah_attr);
502 rqpn = qp->remote_qpn;
503 svc_type = IB_CC_SVCTYPE_RC;
504 break;
505 default:
506 return false;
507 }
508
509 ignore_fecn = is_mcast || (opcode == IB_OPCODE_CNP) ||
510 (opcode == IB_OPCODE_RC_ACKNOWLEDGE);
511
512
513
514
515 do_cnp = prescan ||
516 (opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST &&
517 opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE) ||
518 opcode == TID_OP(READ_RESP) ||
519 opcode == TID_OP(ACK);
520
521
522 if (!ignore_fecn && do_cnp && fecn)
523 hfi1_handle_cnp_tbl[hdr_type](ibp, qp, rqpn, pkey,
524 dlid, rlid, sc, grh);
525
526 if (becn) {
527 u32 lqpn = be32_to_cpu(ohdr->bth[1]) & RVT_QPN_MASK;
528 u8 sl = ibp->sc_to_sl[sc];
529
530 process_becn(ppd, sl, rlid, lqpn, rqpn, svc_type);
531 }
532 return !ignore_fecn && fecn;
533}
534
535struct ps_mdata {
536 struct hfi1_ctxtdata *rcd;
537 u32 rsize;
538 u32 maxcnt;
539 u32 ps_head;
540 u32 ps_tail;
541 u32 ps_seq;
542};
543
544static inline void init_ps_mdata(struct ps_mdata *mdata,
545 struct hfi1_packet *packet)
546{
547 struct hfi1_ctxtdata *rcd = packet->rcd;
548
549 mdata->rcd = rcd;
550 mdata->rsize = packet->rsize;
551 mdata->maxcnt = packet->maxcnt;
552 mdata->ps_head = packet->rhqoff;
553
554 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
555 mdata->ps_tail = get_rcvhdrtail(rcd);
556 if (rcd->ctxt == HFI1_CTRL_CTXT)
557 mdata->ps_seq = rcd->seq_cnt;
558 else
559 mdata->ps_seq = 0;
560 } else {
561 mdata->ps_tail = 0;
562 mdata->ps_seq = rcd->seq_cnt;
563 }
564}
565
566static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
567 struct hfi1_ctxtdata *rcd)
568{
569 if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
570 return mdata->ps_head == mdata->ps_tail;
571 return mdata->ps_seq != rhf_rcv_seq(rhf);
572}
573
574static inline int ps_skip(struct ps_mdata *mdata, u64 rhf,
575 struct hfi1_ctxtdata *rcd)
576{
577
578
579
580
581 if ((rcd->ctxt == HFI1_CTRL_CTXT) && (mdata->ps_head != mdata->ps_tail))
582 return mdata->ps_seq != rhf_rcv_seq(rhf);
583
584 return 0;
585}
586
587static inline void update_ps_mdata(struct ps_mdata *mdata,
588 struct hfi1_ctxtdata *rcd)
589{
590 mdata->ps_head += mdata->rsize;
591 if (mdata->ps_head >= mdata->maxcnt)
592 mdata->ps_head = 0;
593
594
595 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ||
596 (rcd->ctxt == HFI1_CTRL_CTXT)) {
597 if (++mdata->ps_seq > 13)
598 mdata->ps_seq = 1;
599 }
600}
601
602
603
604
605
606
607
608
609
610#define prescan_rxq(rcd, packet) \
611 do { \
612 if (rcd->ppd->cc_prescan) \
613 __prescan_rxq(packet); \
614 } while (0)
615static void __prescan_rxq(struct hfi1_packet *packet)
616{
617 struct hfi1_ctxtdata *rcd = packet->rcd;
618 struct ps_mdata mdata;
619
620 init_ps_mdata(&mdata, packet);
621
622 while (1) {
623 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
624 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
625 packet->rcd->rhf_offset;
626 struct rvt_qp *qp;
627 struct ib_header *hdr;
628 struct rvt_dev_info *rdi = &rcd->dd->verbs_dev.rdi;
629 u64 rhf = rhf_to_cpu(rhf_addr);
630 u32 etype = rhf_rcv_type(rhf), qpn, bth1;
631 u8 lnh;
632
633 if (ps_done(&mdata, rhf, rcd))
634 break;
635
636 if (ps_skip(&mdata, rhf, rcd))
637 goto next;
638
639 if (etype != RHF_RCV_TYPE_IB)
640 goto next;
641
642 packet->hdr = hfi1_get_msgheader(packet->rcd, rhf_addr);
643 hdr = packet->hdr;
644 lnh = ib_get_lnh(hdr);
645
646 if (lnh == HFI1_LRH_BTH) {
647 packet->ohdr = &hdr->u.oth;
648 packet->grh = NULL;
649 } else if (lnh == HFI1_LRH_GRH) {
650 packet->ohdr = &hdr->u.l.oth;
651 packet->grh = &hdr->u.l.grh;
652 } else {
653 goto next;
654 }
655
656 if (!hfi1_may_ecn(packet))
657 goto next;
658
659 bth1 = be32_to_cpu(packet->ohdr->bth[1]);
660 qpn = bth1 & RVT_QPN_MASK;
661 rcu_read_lock();
662 qp = rvt_lookup_qpn(rdi, &ibp->rvp, qpn);
663
664 if (!qp) {
665 rcu_read_unlock();
666 goto next;
667 }
668
669 hfi1_process_ecn_slowpath(qp, packet, true);
670 rcu_read_unlock();
671
672
673 bth1 &= ~(IB_FECN_SMASK | IB_BECN_SMASK);
674 packet->ohdr->bth[1] = cpu_to_be32(bth1);
675next:
676 update_ps_mdata(&mdata, rcd);
677 }
678}
679
680static void process_rcv_qp_work(struct hfi1_packet *packet)
681{
682 struct rvt_qp *qp, *nqp;
683 struct hfi1_ctxtdata *rcd = packet->rcd;
684
685
686
687
688
689 list_for_each_entry_safe(qp, nqp, &rcd->qp_wait_list, rspwait) {
690 list_del_init(&qp->rspwait);
691 if (qp->r_flags & RVT_R_RSP_NAK) {
692 qp->r_flags &= ~RVT_R_RSP_NAK;
693 packet->qp = qp;
694 hfi1_send_rc_ack(packet, 0);
695 }
696 if (qp->r_flags & RVT_R_RSP_SEND) {
697 unsigned long flags;
698
699 qp->r_flags &= ~RVT_R_RSP_SEND;
700 spin_lock_irqsave(&qp->s_lock, flags);
701 if (ib_rvt_state_ops[qp->state] &
702 RVT_PROCESS_OR_FLUSH_SEND)
703 hfi1_schedule_send(qp);
704 spin_unlock_irqrestore(&qp->s_lock, flags);
705 }
706 rvt_put_qp(qp);
707 }
708}
709
710static noinline int max_packet_exceeded(struct hfi1_packet *packet, int thread)
711{
712 if (thread) {
713 if ((packet->numpkt & (MAX_PKT_RECV_THREAD - 1)) == 0)
714
715 process_rcv_qp_work(packet);
716 cond_resched();
717 return RCV_PKT_OK;
718 } else {
719 this_cpu_inc(*packet->rcd->dd->rcv_limit);
720 return RCV_PKT_LIMIT;
721 }
722}
723
724static inline int check_max_packet(struct hfi1_packet *packet, int thread)
725{
726 int ret = RCV_PKT_OK;
727
728 if (unlikely((packet->numpkt & (MAX_PKT_RECV - 1)) == 0))
729 ret = max_packet_exceeded(packet, thread);
730 return ret;
731}
732
733static noinline int skip_rcv_packet(struct hfi1_packet *packet, int thread)
734{
735 int ret;
736
737
738 packet->rhqoff += packet->rsize;
739 if (packet->rhqoff >= packet->maxcnt)
740 packet->rhqoff = 0;
741
742 packet->numpkt++;
743 ret = check_max_packet(packet, thread);
744
745 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
746 packet->rcd->rhf_offset;
747 packet->rhf = rhf_to_cpu(packet->rhf_addr);
748
749 return ret;
750}
751
752static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
753{
754 int ret;
755
756 packet->etype = rhf_rcv_type(packet->rhf);
757
758
759 packet->tlen = rhf_pkt_len(packet->rhf);
760
761 packet->ebuf = NULL;
762 if (rhf_use_egr_bfr(packet->rhf)) {
763 packet->etail = rhf_egr_index(packet->rhf);
764 packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
765 &packet->updegr);
766
767
768
769
770
771 prefetch_range(packet->ebuf,
772 packet->tlen - ((packet->rcd->rcvhdrqentsize -
773 (rhf_hdrq_offset(packet->rhf)
774 + 2)) * 4));
775 }
776
777
778
779
780
781
782
783
784
785 packet->rcd->rhf_rcv_function_map[packet->etype](packet);
786 packet->numpkt++;
787
788
789 packet->rhqoff += packet->rsize;
790 if (packet->rhqoff >= packet->maxcnt)
791 packet->rhqoff = 0;
792
793 ret = check_max_packet(packet, thread);
794
795 packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
796 packet->rcd->rhf_offset;
797 packet->rhf = rhf_to_cpu(packet->rhf_addr);
798
799 return ret;
800}
801
802static inline void process_rcv_update(int last, struct hfi1_packet *packet)
803{
804
805
806
807
808
809
810 if (!last && !(packet->numpkt & 0xf)) {
811 update_usrhead(packet->rcd, packet->rhqoff, packet->updegr,
812 packet->etail, 0, 0);
813 packet->updegr = 0;
814 }
815 packet->grh = NULL;
816}
817
818static inline void finish_packet(struct hfi1_packet *packet)
819{
820
821
822
823
824
825
826 update_usrhead(packet->rcd, packet->rcd->head, packet->updegr,
827 packet->etail, rcv_intr_dynamic, packet->numpkt);
828}
829
830
831
832
833int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
834{
835 u32 seq;
836 int last = RCV_PKT_OK;
837 struct hfi1_packet packet;
838
839 init_packet(rcd, &packet);
840 seq = rhf_rcv_seq(packet.rhf);
841 if (seq != rcd->seq_cnt) {
842 last = RCV_PKT_DONE;
843 goto bail;
844 }
845
846 prescan_rxq(rcd, &packet);
847
848 while (last == RCV_PKT_OK) {
849 last = process_rcv_packet(&packet, thread);
850 seq = rhf_rcv_seq(packet.rhf);
851 if (++rcd->seq_cnt > 13)
852 rcd->seq_cnt = 1;
853 if (seq != rcd->seq_cnt)
854 last = RCV_PKT_DONE;
855 process_rcv_update(last, &packet);
856 }
857 process_rcv_qp_work(&packet);
858 rcd->head = packet.rhqoff;
859bail:
860 finish_packet(&packet);
861 return last;
862}
863
864int handle_receive_interrupt_dma_rtail(struct hfi1_ctxtdata *rcd, int thread)
865{
866 u32 hdrqtail;
867 int last = RCV_PKT_OK;
868 struct hfi1_packet packet;
869
870 init_packet(rcd, &packet);
871 hdrqtail = get_rcvhdrtail(rcd);
872 if (packet.rhqoff == hdrqtail) {
873 last = RCV_PKT_DONE;
874 goto bail;
875 }
876 smp_rmb();
877
878 prescan_rxq(rcd, &packet);
879
880 while (last == RCV_PKT_OK) {
881 last = process_rcv_packet(&packet, thread);
882 if (packet.rhqoff == hdrqtail)
883 last = RCV_PKT_DONE;
884 process_rcv_update(last, &packet);
885 }
886 process_rcv_qp_work(&packet);
887 rcd->head = packet.rhqoff;
888bail:
889 finish_packet(&packet);
890 return last;
891}
892
893static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt)
894{
895 struct hfi1_ctxtdata *rcd;
896 u16 i;
897
898
899
900
901
902
903 if (ctxt >= dd->first_dyn_alloc_ctxt) {
904 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
905 if (rcd) {
906 rcd->do_interrupt =
907 &handle_receive_interrupt_nodma_rtail;
908 hfi1_rcd_put(rcd);
909 }
910 return;
911 }
912
913 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
914 rcd = hfi1_rcd_get_by_index(dd, i);
915 if (rcd)
916 rcd->do_interrupt =
917 &handle_receive_interrupt_nodma_rtail;
918 hfi1_rcd_put(rcd);
919 }
920}
921
922static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt)
923{
924 struct hfi1_ctxtdata *rcd;
925 u16 i;
926
927
928
929
930
931
932 if (ctxt >= dd->first_dyn_alloc_ctxt) {
933 rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
934 if (rcd) {
935 rcd->do_interrupt =
936 &handle_receive_interrupt_dma_rtail;
937 hfi1_rcd_put(rcd);
938 }
939 return;
940 }
941
942 for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
943 rcd = hfi1_rcd_get_by_index(dd, i);
944 if (rcd)
945 rcd->do_interrupt =
946 &handle_receive_interrupt_dma_rtail;
947 hfi1_rcd_put(rcd);
948 }
949}
950
951void set_all_slowpath(struct hfi1_devdata *dd)
952{
953 struct hfi1_ctxtdata *rcd;
954 u16 i;
955
956
957 for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
958 rcd = hfi1_rcd_get_by_index(dd, i);
959 if (!rcd)
960 continue;
961 if (i < dd->first_dyn_alloc_ctxt || rcd->is_vnic)
962 rcd->do_interrupt = &handle_receive_interrupt;
963
964 hfi1_rcd_put(rcd);
965 }
966}
967
968static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
969 struct hfi1_packet *packet,
970 struct hfi1_devdata *dd)
971{
972 struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
973 u8 etype = rhf_rcv_type(packet->rhf);
974 u8 sc = SC15_PACKET;
975
976 if (etype == RHF_RCV_TYPE_IB) {
977 struct ib_header *hdr = hfi1_get_msgheader(packet->rcd,
978 packet->rhf_addr);
979 sc = hfi1_9B_get_sc5(hdr, packet->rhf);
980 } else if (etype == RHF_RCV_TYPE_BYPASS) {
981 struct hfi1_16b_header *hdr = hfi1_get_16B_header(
982 packet->rcd,
983 packet->rhf_addr);
984 sc = hfi1_16B_get_sc(hdr);
985 }
986 if (sc != SC15_PACKET) {
987 int hwstate = driver_lstate(rcd->ppd);
988
989 if (hwstate != IB_PORT_ACTIVE) {
990 dd_dev_info(dd,
991 "Unexpected link state %s\n",
992 opa_lstate_name(hwstate));
993 return 0;
994 }
995
996 queue_work(rcd->ppd->link_wq, lsaw);
997 return 1;
998 }
999 return 0;
1000}
1001
1002
1003
1004
1005
1006
1007
1008
1009int handle_receive_interrupt(struct hfi1_ctxtdata *rcd, int thread)
1010{
1011 struct hfi1_devdata *dd = rcd->dd;
1012 u32 hdrqtail;
1013 int needset, last = RCV_PKT_OK;
1014 struct hfi1_packet packet;
1015 int skip_pkt = 0;
1016
1017
1018 needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
1019
1020 init_packet(rcd, &packet);
1021
1022 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1023 u32 seq = rhf_rcv_seq(packet.rhf);
1024
1025 if (seq != rcd->seq_cnt) {
1026 last = RCV_PKT_DONE;
1027 goto bail;
1028 }
1029 hdrqtail = 0;
1030 } else {
1031 hdrqtail = get_rcvhdrtail(rcd);
1032 if (packet.rhqoff == hdrqtail) {
1033 last = RCV_PKT_DONE;
1034 goto bail;
1035 }
1036 smp_rmb();
1037
1038
1039
1040
1041
1042 if (rcd->ctxt == HFI1_CTRL_CTXT) {
1043 u32 seq = rhf_rcv_seq(packet.rhf);
1044
1045 if (seq != rcd->seq_cnt)
1046 skip_pkt = 1;
1047 }
1048 }
1049
1050 prescan_rxq(rcd, &packet);
1051
1052 while (last == RCV_PKT_OK) {
1053 if (unlikely(dd->do_drop &&
1054 atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
1055 DROP_PACKET_ON)) {
1056 dd->do_drop = 0;
1057
1058
1059 packet.rhqoff += packet.rsize;
1060 packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
1061 packet.rhqoff +
1062 rcd->rhf_offset;
1063 packet.rhf = rhf_to_cpu(packet.rhf_addr);
1064
1065 } else if (skip_pkt) {
1066 last = skip_rcv_packet(&packet, thread);
1067 skip_pkt = 0;
1068 } else {
1069
1070 if (unlikely(rcd->ppd->host_link_state ==
1071 HLS_UP_ARMED) &&
1072 set_armed_to_active(rcd, &packet, dd))
1073 goto bail;
1074 last = process_rcv_packet(&packet, thread);
1075 }
1076
1077 if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1078 u32 seq = rhf_rcv_seq(packet.rhf);
1079
1080 if (++rcd->seq_cnt > 13)
1081 rcd->seq_cnt = 1;
1082 if (seq != rcd->seq_cnt)
1083 last = RCV_PKT_DONE;
1084 if (needset) {
1085 dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");
1086 set_nodma_rtail(dd, rcd->ctxt);
1087 needset = 0;
1088 }
1089 } else {
1090 if (packet.rhqoff == hdrqtail)
1091 last = RCV_PKT_DONE;
1092
1093
1094
1095
1096 if (rcd->ctxt == HFI1_CTRL_CTXT) {
1097 u32 seq = rhf_rcv_seq(packet.rhf);
1098
1099 if (++rcd->seq_cnt > 13)
1100 rcd->seq_cnt = 1;
1101 if (!last && (seq != rcd->seq_cnt))
1102 skip_pkt = 1;
1103 }
1104
1105 if (needset) {
1106 dd_dev_info(dd,
1107 "Switching to DMA_RTAIL\n");
1108 set_dma_rtail(dd, rcd->ctxt);
1109 needset = 0;
1110 }
1111 }
1112
1113 process_rcv_update(last, &packet);
1114 }
1115
1116 process_rcv_qp_work(&packet);
1117 rcd->head = packet.rhqoff;
1118
1119bail:
1120
1121
1122
1123
1124 finish_packet(&packet);
1125 return last;
1126}
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145void receive_interrupt_work(struct work_struct *work)
1146{
1147 struct hfi1_pportdata *ppd = container_of(work, struct hfi1_pportdata,
1148 linkstate_active_work);
1149 struct hfi1_devdata *dd = ppd->dd;
1150 struct hfi1_ctxtdata *rcd;
1151 u16 i;
1152
1153
1154 ppd->neighbor_normal = 1;
1155 set_link_state(ppd, HLS_UP_ACTIVE);
1156
1157
1158
1159
1160
1161 for (i = HFI1_CTRL_CTXT; i < dd->first_dyn_alloc_ctxt; i++) {
1162 rcd = hfi1_rcd_get_by_index(dd, i);
1163 if (rcd)
1164 force_recv_intr(rcd);
1165 hfi1_rcd_put(rcd);
1166 }
1167}
1168
1169
1170
1171
1172
1173int mtu_to_enum(u32 mtu, int default_if_bad)
1174{
1175 switch (mtu) {
1176 case 0: return OPA_MTU_0;
1177 case 256: return OPA_MTU_256;
1178 case 512: return OPA_MTU_512;
1179 case 1024: return OPA_MTU_1024;
1180 case 2048: return OPA_MTU_2048;
1181 case 4096: return OPA_MTU_4096;
1182 case 8192: return OPA_MTU_8192;
1183 case 10240: return OPA_MTU_10240;
1184 }
1185 return default_if_bad;
1186}
1187
1188u16 enum_to_mtu(int mtu)
1189{
1190 switch (mtu) {
1191 case OPA_MTU_0: return 0;
1192 case OPA_MTU_256: return 256;
1193 case OPA_MTU_512: return 512;
1194 case OPA_MTU_1024: return 1024;
1195 case OPA_MTU_2048: return 2048;
1196 case OPA_MTU_4096: return 4096;
1197 case OPA_MTU_8192: return 8192;
1198 case OPA_MTU_10240: return 10240;
1199 default: return 0xffff;
1200 }
1201}
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211int set_mtu(struct hfi1_pportdata *ppd)
1212{
1213 struct hfi1_devdata *dd = ppd->dd;
1214 int i, drain, ret = 0, is_up = 0;
1215
1216 ppd->ibmtu = 0;
1217 for (i = 0; i < ppd->vls_supported; i++)
1218 if (ppd->ibmtu < dd->vld[i].mtu)
1219 ppd->ibmtu = dd->vld[i].mtu;
1220 ppd->ibmaxlen = ppd->ibmtu + lrh_max_header_bytes(ppd->dd);
1221
1222 mutex_lock(&ppd->hls_lock);
1223 if (ppd->host_link_state == HLS_UP_INIT ||
1224 ppd->host_link_state == HLS_UP_ARMED ||
1225 ppd->host_link_state == HLS_UP_ACTIVE)
1226 is_up = 1;
1227
1228 drain = !is_ax(dd) && is_up;
1229
1230 if (drain)
1231
1232
1233
1234
1235
1236 ret = stop_drain_data_vls(dd);
1237
1238 if (ret) {
1239 dd_dev_err(dd, "%s: cannot stop/drain VLs - refusing to change per-VL MTUs\n",
1240 __func__);
1241 goto err;
1242 }
1243
1244 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_MTU, 0);
1245
1246 if (drain)
1247 open_fill_data_vls(dd);
1248
1249err:
1250 mutex_unlock(&ppd->hls_lock);
1251
1252 return ret;
1253}
1254
1255int hfi1_set_lid(struct hfi1_pportdata *ppd, u32 lid, u8 lmc)
1256{
1257 struct hfi1_devdata *dd = ppd->dd;
1258
1259 ppd->lid = lid;
1260 ppd->lmc = lmc;
1261 hfi1_set_ib_cfg(ppd, HFI1_IB_CFG_LIDLMC, 0);
1262
1263 dd_dev_info(dd, "port %u: got a lid: 0x%x\n", ppd->port, lid);
1264
1265 return 0;
1266}
1267
1268void shutdown_led_override(struct hfi1_pportdata *ppd)
1269{
1270 struct hfi1_devdata *dd = ppd->dd;
1271
1272
1273
1274
1275
1276
1277 smp_rmb();
1278 if (atomic_read(&ppd->led_override_timer_active)) {
1279 del_timer_sync(&ppd->led_override_timer);
1280 atomic_set(&ppd->led_override_timer_active, 0);
1281
1282 smp_wmb();
1283 }
1284
1285
1286 write_csr(dd, DCC_CFG_LED_CNTRL, 0);
1287}
1288
1289static void run_led_override(struct timer_list *t)
1290{
1291 struct hfi1_pportdata *ppd = from_timer(ppd, t, led_override_timer);
1292 struct hfi1_devdata *dd = ppd->dd;
1293 unsigned long timeout;
1294 int phase_idx;
1295
1296 if (!(dd->flags & HFI1_INITTED))
1297 return;
1298
1299 phase_idx = ppd->led_override_phase & 1;
1300
1301 setextled(dd, phase_idx);
1302
1303 timeout = ppd->led_override_vals[phase_idx];
1304
1305
1306 ppd->led_override_phase = !ppd->led_override_phase;
1307
1308 mod_timer(&ppd->led_override_timer, jiffies + timeout);
1309}
1310
1311
1312
1313
1314
1315
1316
1317void hfi1_start_led_override(struct hfi1_pportdata *ppd, unsigned int timeon,
1318 unsigned int timeoff)
1319{
1320 if (!(ppd->dd->flags & HFI1_INITTED))
1321 return;
1322
1323
1324 ppd->led_override_vals[0] = msecs_to_jiffies(timeoff);
1325 ppd->led_override_vals[1] = msecs_to_jiffies(timeon);
1326
1327
1328 ppd->led_override_phase = 1;
1329
1330
1331
1332
1333
1334 if (!timer_pending(&ppd->led_override_timer)) {
1335 timer_setup(&ppd->led_override_timer, run_led_override, 0);
1336 ppd->led_override_timer.expires = jiffies + 1;
1337 add_timer(&ppd->led_override_timer);
1338 atomic_set(&ppd->led_override_timer_active, 1);
1339
1340 smp_wmb();
1341 }
1342}
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353int hfi1_reset_device(int unit)
1354{
1355 int ret;
1356 struct hfi1_devdata *dd = hfi1_lookup(unit);
1357 struct hfi1_pportdata *ppd;
1358 int pidx;
1359
1360 if (!dd) {
1361 ret = -ENODEV;
1362 goto bail;
1363 }
1364
1365 dd_dev_info(dd, "Reset on unit %u requested\n", unit);
1366
1367 if (!dd->kregbase1 || !(dd->flags & HFI1_PRESENT)) {
1368 dd_dev_info(dd,
1369 "Invalid unit number %u or not initialized or not present\n",
1370 unit);
1371 ret = -ENXIO;
1372 goto bail;
1373 }
1374
1375
1376 mutex_lock(&hfi1_mutex);
1377 if (dd->rcd)
1378 if (hfi1_stats.sps_ctxts) {
1379 mutex_unlock(&hfi1_mutex);
1380 ret = -EBUSY;
1381 goto bail;
1382 }
1383 mutex_unlock(&hfi1_mutex);
1384
1385 for (pidx = 0; pidx < dd->num_pports; ++pidx) {
1386 ppd = dd->pport + pidx;
1387
1388 shutdown_led_override(ppd);
1389 }
1390 if (dd->flags & HFI1_HAS_SEND_DMA)
1391 sdma_exit(dd);
1392
1393 hfi1_reset_cpu_counters(dd);
1394
1395 ret = hfi1_init(dd, 1);
1396
1397 if (ret)
1398 dd_dev_err(dd,
1399 "Reinitialize unit %u after reset failed with %d\n",
1400 unit, ret);
1401 else
1402 dd_dev_info(dd, "Reinitialized unit %u after resetting\n",
1403 unit);
1404
1405bail:
1406 return ret;
1407}
1408
1409static inline void hfi1_setup_ib_header(struct hfi1_packet *packet)
1410{
1411 packet->hdr = (struct hfi1_ib_message_header *)
1412 hfi1_get_msgheader(packet->rcd,
1413 packet->rhf_addr);
1414 packet->hlen = (u8 *)packet->rhf_addr - (u8 *)packet->hdr;
1415}
1416
1417static int hfi1_bypass_ingress_pkt_check(struct hfi1_packet *packet)
1418{
1419 struct hfi1_pportdata *ppd = packet->rcd->ppd;
1420
1421
1422 if ((!packet->slid) || (!packet->dlid))
1423 return -EINVAL;
1424
1425
1426 if ((!(hfi1_is_16B_mcast(packet->dlid))) &&
1427 (packet->dlid !=
1428 opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) {
1429 if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid)
1430 return -EINVAL;
1431 }
1432
1433
1434 if ((hfi1_is_16B_mcast(packet->dlid)) && (packet->sc == 0xF))
1435 return -EINVAL;
1436
1437
1438 if ((packet->dlid == opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE),
1439 16B)) &&
1440 (packet->sc != 0xF))
1441 return -EINVAL;
1442
1443 return 0;
1444}
1445
1446static int hfi1_setup_9B_packet(struct hfi1_packet *packet)
1447{
1448 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
1449 struct ib_header *hdr;
1450 u8 lnh;
1451
1452 hfi1_setup_ib_header(packet);
1453 hdr = packet->hdr;
1454
1455 lnh = ib_get_lnh(hdr);
1456 if (lnh == HFI1_LRH_BTH) {
1457 packet->ohdr = &hdr->u.oth;
1458 packet->grh = NULL;
1459 } else if (lnh == HFI1_LRH_GRH) {
1460 u32 vtf;
1461
1462 packet->ohdr = &hdr->u.l.oth;
1463 packet->grh = &hdr->u.l.grh;
1464 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
1465 goto drop;
1466 vtf = be32_to_cpu(packet->grh->version_tclass_flow);
1467 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
1468 goto drop;
1469 } else {
1470 goto drop;
1471 }
1472
1473
1474 packet->payload = packet->ebuf;
1475 packet->opcode = ib_bth_get_opcode(packet->ohdr);
1476 packet->slid = ib_get_slid(hdr);
1477 packet->dlid = ib_get_dlid(hdr);
1478 if (unlikely((packet->dlid >= be16_to_cpu(IB_MULTICAST_LID_BASE)) &&
1479 (packet->dlid != be16_to_cpu(IB_LID_PERMISSIVE))))
1480 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
1481 be16_to_cpu(IB_MULTICAST_LID_BASE);
1482 packet->sl = ib_get_sl(hdr);
1483 packet->sc = hfi1_9B_get_sc5(hdr, packet->rhf);
1484 packet->pad = ib_bth_get_pad(packet->ohdr);
1485 packet->extra_byte = 0;
1486 packet->pkey = ib_bth_get_pkey(packet->ohdr);
1487 packet->migrated = ib_bth_is_migration(packet->ohdr);
1488
1489 return 0;
1490drop:
1491 ibp->rvp.n_pkt_drops++;
1492 return -EINVAL;
1493}
1494
1495static int hfi1_setup_bypass_packet(struct hfi1_packet *packet)
1496{
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507 struct hfi1_ctxtdata *rcd = packet->rcd;
1508 struct hfi1_pportdata *ppd = rcd->ppd;
1509 struct hfi1_ibport *ibp = &ppd->ibport_data;
1510 u8 l4;
1511
1512 packet->hdr = (struct hfi1_16b_header *)
1513 hfi1_get_16B_header(packet->rcd,
1514 packet->rhf_addr);
1515 l4 = hfi1_16B_get_l4(packet->hdr);
1516 if (l4 == OPA_16B_L4_IB_LOCAL) {
1517 packet->ohdr = packet->ebuf;
1518 packet->grh = NULL;
1519 packet->opcode = ib_bth_get_opcode(packet->ohdr);
1520 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
1521
1522 packet->hlen = hdr_len_by_opcode[packet->opcode] +
1523 (LRH_16B_BYTES - LRH_9B_BYTES);
1524 packet->migrated = opa_bth_is_migration(packet->ohdr);
1525 } else if (l4 == OPA_16B_L4_IB_GLOBAL) {
1526 u32 vtf;
1527 u8 grh_len = sizeof(struct ib_grh);
1528
1529 packet->ohdr = packet->ebuf + grh_len;
1530 packet->grh = packet->ebuf;
1531 packet->opcode = ib_bth_get_opcode(packet->ohdr);
1532 packet->pad = hfi1_16B_bth_get_pad(packet->ohdr);
1533
1534 packet->hlen = hdr_len_by_opcode[packet->opcode] +
1535 (LRH_16B_BYTES - LRH_9B_BYTES) + grh_len;
1536 packet->migrated = opa_bth_is_migration(packet->ohdr);
1537
1538 if (packet->grh->next_hdr != IB_GRH_NEXT_HDR)
1539 goto drop;
1540 vtf = be32_to_cpu(packet->grh->version_tclass_flow);
1541 if ((vtf >> IB_GRH_VERSION_SHIFT) != IB_GRH_VERSION)
1542 goto drop;
1543 } else if (l4 == OPA_16B_L4_FM) {
1544 packet->mgmt = packet->ebuf;
1545 packet->ohdr = NULL;
1546 packet->grh = NULL;
1547 packet->opcode = IB_OPCODE_UD_SEND_ONLY;
1548 packet->pad = OPA_16B_L4_FM_PAD;
1549 packet->hlen = OPA_16B_L4_FM_HLEN;
1550 packet->migrated = false;
1551 } else {
1552 goto drop;
1553 }
1554
1555
1556 packet->payload = packet->ebuf + packet->hlen - LRH_16B_BYTES;
1557 packet->slid = hfi1_16B_get_slid(packet->hdr);
1558 packet->dlid = hfi1_16B_get_dlid(packet->hdr);
1559 if (unlikely(hfi1_is_16B_mcast(packet->dlid)))
1560 packet->dlid += opa_get_mcast_base(OPA_MCAST_NR) -
1561 opa_get_lid(opa_get_mcast_base(OPA_MCAST_NR),
1562 16B);
1563 packet->sc = hfi1_16B_get_sc(packet->hdr);
1564 packet->sl = ibp->sc_to_sl[packet->sc];
1565 packet->extra_byte = SIZE_OF_LT;
1566 packet->pkey = hfi1_16B_get_pkey(packet->hdr);
1567
1568 if (hfi1_bypass_ingress_pkt_check(packet))
1569 goto drop;
1570
1571 return 0;
1572drop:
1573 hfi1_cdbg(PKT, "%s: packet dropped\n", __func__);
1574 ibp->rvp.n_pkt_drops++;
1575 return -EINVAL;
1576}
1577
1578static void show_eflags_errs(struct hfi1_packet *packet)
1579{
1580 struct hfi1_ctxtdata *rcd = packet->rcd;
1581 u32 rte = rhf_rcv_type_err(packet->rhf);
1582
1583 dd_dev_err(rcd->dd,
1584 "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s] rte 0x%x\n",
1585 rcd->ctxt, packet->rhf,
1586 packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
1587 packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
1588 packet->rhf & RHF_DC_ERR ? "dc " : "",
1589 packet->rhf & RHF_TID_ERR ? "tid " : "",
1590 packet->rhf & RHF_LEN_ERR ? "len " : "",
1591 packet->rhf & RHF_ECC_ERR ? "ecc " : "",
1592 packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
1593 rte);
1594}
1595
1596void handle_eflags(struct hfi1_packet *packet)
1597{
1598 struct hfi1_ctxtdata *rcd = packet->rcd;
1599
1600 rcv_hdrerr(rcd, rcd->ppd, packet);
1601 if (rhf_err_flags(packet->rhf))
1602 show_eflags_errs(packet);
1603}
1604
1605
1606
1607
1608
1609static int process_receive_ib(struct hfi1_packet *packet)
1610{
1611 if (hfi1_setup_9B_packet(packet))
1612 return RHF_RCV_CONTINUE;
1613
1614 if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1615 return RHF_RCV_CONTINUE;
1616
1617 trace_hfi1_rcvhdr(packet);
1618
1619 if (unlikely(rhf_err_flags(packet->rhf))) {
1620 handle_eflags(packet);
1621 return RHF_RCV_CONTINUE;
1622 }
1623
1624 hfi1_ib_rcv(packet);
1625 return RHF_RCV_CONTINUE;
1626}
1627
1628static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet)
1629{
1630
1631 if (packet->rcd->is_vnic)
1632 return true;
1633
1634 if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) &&
1635 (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR))
1636 return true;
1637
1638 return false;
1639}
1640
1641static int process_receive_bypass(struct hfi1_packet *packet)
1642{
1643 struct hfi1_devdata *dd = packet->rcd->dd;
1644
1645 if (hfi1_is_vnic_packet(packet)) {
1646 hfi1_vnic_bypass_rcv(packet);
1647 return RHF_RCV_CONTINUE;
1648 }
1649
1650 if (hfi1_setup_bypass_packet(packet))
1651 return RHF_RCV_CONTINUE;
1652
1653 trace_hfi1_rcvhdr(packet);
1654
1655 if (unlikely(rhf_err_flags(packet->rhf))) {
1656 handle_eflags(packet);
1657 return RHF_RCV_CONTINUE;
1658 }
1659
1660 if (hfi1_16B_get_l2(packet->hdr) == 0x2) {
1661 hfi1_16B_rcv(packet);
1662 } else {
1663 dd_dev_err(dd,
1664 "Bypass packets other than 16B are not supported in normal operation. Dropping\n");
1665 incr_cntr64(&dd->sw_rcv_bypass_packet_errors);
1666 if (!(dd->err_info_rcvport.status_and_code &
1667 OPA_EI_STATUS_SMASK)) {
1668 u64 *flits = packet->ebuf;
1669
1670 if (flits && !(packet->rhf & RHF_LEN_ERR)) {
1671 dd->err_info_rcvport.packet_flit1 = flits[0];
1672 dd->err_info_rcvport.packet_flit2 =
1673 packet->tlen > sizeof(flits[0]) ?
1674 flits[1] : 0;
1675 }
1676 dd->err_info_rcvport.status_and_code |=
1677 (OPA_EI_STATUS_SMASK | BAD_L2_ERR);
1678 }
1679 }
1680 return RHF_RCV_CONTINUE;
1681}
1682
1683static int process_receive_error(struct hfi1_packet *packet)
1684{
1685
1686 if (unlikely(
1687 hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
1688 (rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR ||
1689 packet->rhf & RHF_DC_ERR)))
1690 return RHF_RCV_CONTINUE;
1691
1692 hfi1_setup_ib_header(packet);
1693 handle_eflags(packet);
1694
1695 if (unlikely(rhf_err_flags(packet->rhf)))
1696 dd_dev_err(packet->rcd->dd,
1697 "Unhandled error packet received. Dropping.\n");
1698
1699 return RHF_RCV_CONTINUE;
1700}
1701
1702static int kdeth_process_expected(struct hfi1_packet *packet)
1703{
1704 hfi1_setup_9B_packet(packet);
1705 if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1706 return RHF_RCV_CONTINUE;
1707
1708 if (unlikely(rhf_err_flags(packet->rhf))) {
1709 struct hfi1_ctxtdata *rcd = packet->rcd;
1710
1711 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
1712 return RHF_RCV_CONTINUE;
1713 }
1714
1715 hfi1_kdeth_expected_rcv(packet);
1716 return RHF_RCV_CONTINUE;
1717}
1718
1719static int kdeth_process_eager(struct hfi1_packet *packet)
1720{
1721 hfi1_setup_9B_packet(packet);
1722 if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1723 return RHF_RCV_CONTINUE;
1724
1725 trace_hfi1_rcvhdr(packet);
1726 if (unlikely(rhf_err_flags(packet->rhf))) {
1727 struct hfi1_ctxtdata *rcd = packet->rcd;
1728
1729 show_eflags_errs(packet);
1730 if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
1731 return RHF_RCV_CONTINUE;
1732 }
1733
1734 hfi1_kdeth_eager_rcv(packet);
1735 return RHF_RCV_CONTINUE;
1736}
1737
1738static int process_receive_invalid(struct hfi1_packet *packet)
1739{
1740 dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1741 rhf_rcv_type(packet->rhf));
1742 return RHF_RCV_CONTINUE;
1743}
1744
1745void seqfile_dump_rcd(struct seq_file *s, struct hfi1_ctxtdata *rcd)
1746{
1747 struct hfi1_packet packet;
1748 struct ps_mdata mdata;
1749
1750 seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n",
1751 rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize,
1752 HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
1753 "dma_rtail" : "nodma_rtail",
1754 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &
1755 RCV_HDR_HEAD_HEAD_MASK,
1756 read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL));
1757
1758 init_packet(rcd, &packet);
1759 init_ps_mdata(&mdata, &packet);
1760
1761 while (1) {
1762 __le32 *rhf_addr = (__le32 *)rcd->rcvhdrq + mdata.ps_head +
1763 rcd->rhf_offset;
1764 struct ib_header *hdr;
1765 u64 rhf = rhf_to_cpu(rhf_addr);
1766 u32 etype = rhf_rcv_type(rhf), qpn;
1767 u8 opcode;
1768 u32 psn;
1769 u8 lnh;
1770
1771 if (ps_done(&mdata, rhf, rcd))
1772 break;
1773
1774 if (ps_skip(&mdata, rhf, rcd))
1775 goto next;
1776
1777 if (etype > RHF_RCV_TYPE_IB)
1778 goto next;
1779
1780 packet.hdr = hfi1_get_msgheader(rcd, rhf_addr);
1781 hdr = packet.hdr;
1782
1783 lnh = be16_to_cpu(hdr->lrh[0]) & 3;
1784
1785 if (lnh == HFI1_LRH_BTH)
1786 packet.ohdr = &hdr->u.oth;
1787 else if (lnh == HFI1_LRH_GRH)
1788 packet.ohdr = &hdr->u.l.oth;
1789 else
1790 goto next;
1791
1792 opcode = (be32_to_cpu(packet.ohdr->bth[0]) >> 24);
1793 qpn = be32_to_cpu(packet.ohdr->bth[1]) & RVT_QPN_MASK;
1794 psn = mask_psn(be32_to_cpu(packet.ohdr->bth[2]));
1795
1796 seq_printf(s, "\tEnt %u: opcode 0x%x, qpn 0x%x, psn 0x%x\n",
1797 mdata.ps_head, opcode, qpn, psn);
1798next:
1799 update_ps_mdata(&mdata, rcd);
1800 }
1801}
1802
1803const rhf_rcv_function_ptr normal_rhf_rcv_functions[] = {
1804 [RHF_RCV_TYPE_EXPECTED] = kdeth_process_expected,
1805 [RHF_RCV_TYPE_EAGER] = kdeth_process_eager,
1806 [RHF_RCV_TYPE_IB] = process_receive_ib,
1807 [RHF_RCV_TYPE_ERROR] = process_receive_error,
1808 [RHF_RCV_TYPE_BYPASS] = process_receive_bypass,
1809 [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
1810 [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
1811 [RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
1812};
1813