1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#include <linux/io.h>
49#include <rdma/rdma_vt.h>
50#include <rdma/rdmavt_qp.h>
51
52#include "hfi.h"
53#include "qp.h"
54#include "rc.h"
55#include "verbs_txreq.h"
56#include "trace.h"
57
58struct rvt_ack_entry *find_prev_entry(struct rvt_qp *qp, u32 psn, u8 *prev,
59 u8 *prev_ack, bool *scheduled)
60 __must_hold(&qp->s_lock)
61{
62 struct rvt_ack_entry *e = NULL;
63 u8 i, p;
64 bool s = true;
65
66 for (i = qp->r_head_ack_queue; ; i = p) {
67 if (i == qp->s_tail_ack_queue)
68 s = false;
69 if (i)
70 p = i - 1;
71 else
72 p = rvt_size_atomic(ib_to_rvt(qp->ibqp.device));
73 if (p == qp->r_head_ack_queue) {
74 e = NULL;
75 break;
76 }
77 e = &qp->s_ack_queue[p];
78 if (!e->opcode) {
79 e = NULL;
80 break;
81 }
82 if (cmp_psn(psn, e->psn) >= 0) {
83 if (p == qp->s_tail_ack_queue &&
84 cmp_psn(psn, e->lpsn) <= 0)
85 s = false;
86 break;
87 }
88 }
89 if (prev)
90 *prev = p;
91 if (prev_ack)
92 *prev_ack = i;
93 if (scheduled)
94 *scheduled = s;
95 return e;
96}
97
98
99
100
101
102
103
104
105
106
107
108
109static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
110 struct ib_other_headers *ohdr,
111 struct hfi1_pkt_state *ps)
112{
113 struct rvt_ack_entry *e;
114 u32 hwords, hdrlen;
115 u32 len = 0;
116 u32 bth0 = 0, bth2 = 0;
117 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT);
118 int middle = 0;
119 u32 pmtu = qp->pmtu;
120 struct hfi1_qp_priv *qpriv = qp->priv;
121 bool last_pkt;
122 u32 delta;
123 u8 next = qp->s_tail_ack_queue;
124 struct tid_rdma_request *req;
125
126 trace_hfi1_rsp_make_rc_ack(qp, 0);
127 lockdep_assert_held(&qp->s_lock);
128
129 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
130 goto bail;
131
132 if (qpriv->hdr_type == HFI1_PKT_TYPE_9B)
133
134 hwords = 5;
135 else
136
137 hwords = 7;
138
139 switch (qp->s_ack_state) {
140 case OP(RDMA_READ_RESPONSE_LAST):
141 case OP(RDMA_READ_RESPONSE_ONLY):
142 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
143 release_rdma_sge_mr(e);
144
145 case OP(ATOMIC_ACKNOWLEDGE):
146
147
148
149
150
151 if (++next > rvt_size_atomic(&dev->rdi))
152 next = 0;
153
154
155
156
157 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
158 if (e->opcode != TID_OP(WRITE_REQ) &&
159 qp->s_acked_ack_queue == qp->s_tail_ack_queue)
160 qp->s_acked_ack_queue = next;
161 qp->s_tail_ack_queue = next;
162 trace_hfi1_rsp_make_rc_ack(qp, e->psn);
163
164 case OP(SEND_ONLY):
165 case OP(ACKNOWLEDGE):
166
167 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
168 if (qp->s_flags & RVT_S_ACK_PENDING)
169 goto normal;
170 goto bail;
171 }
172
173 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
174
175 if ((qpriv->s_flags & HFI1_R_TID_WAIT_INTERLCK) ||
176 hfi1_tid_rdma_ack_interlock(qp, e)) {
177 iowait_set_flag(&qpriv->s_iowait, IOWAIT_PENDING_IB);
178 goto bail;
179 }
180 if (e->opcode == OP(RDMA_READ_REQUEST)) {
181
182
183
184
185
186
187 len = e->rdma_sge.sge_length;
188 if (len && !e->rdma_sge.mr) {
189 if (qp->s_acked_ack_queue ==
190 qp->s_tail_ack_queue)
191 qp->s_acked_ack_queue =
192 qp->r_head_ack_queue;
193 qp->s_tail_ack_queue = qp->r_head_ack_queue;
194 goto bail;
195 }
196
197 ps->s_txreq->mr = e->rdma_sge.mr;
198 if (ps->s_txreq->mr)
199 rvt_get_mr(ps->s_txreq->mr);
200 qp->s_ack_rdma_sge.sge = e->rdma_sge;
201 qp->s_ack_rdma_sge.num_sge = 1;
202 ps->s_txreq->ss = &qp->s_ack_rdma_sge;
203 if (len > pmtu) {
204 len = pmtu;
205 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
206 } else {
207 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
208 e->sent = 1;
209 }
210 ohdr->u.aeth = rvt_compute_aeth(qp);
211 hwords++;
212 qp->s_ack_rdma_psn = e->psn;
213 bth2 = mask_psn(qp->s_ack_rdma_psn++);
214 } else if (e->opcode == TID_OP(WRITE_REQ)) {
215
216
217
218
219
220
221
222 req = ack_to_tid_req(e);
223 if (req->state == TID_REQUEST_RESEND ||
224 req->state == TID_REQUEST_INIT_RESEND)
225 goto bail;
226 qp->s_ack_state = TID_OP(WRITE_RESP);
227 qp->s_ack_rdma_psn = mask_psn(e->psn + req->cur_seg);
228 goto write_resp;
229 } else if (e->opcode == TID_OP(READ_REQ)) {
230
231
232
233
234
235
236 len = e->rdma_sge.sge_length;
237 if (len && !e->rdma_sge.mr) {
238 if (qp->s_acked_ack_queue ==
239 qp->s_tail_ack_queue)
240 qp->s_acked_ack_queue =
241 qp->r_head_ack_queue;
242 qp->s_tail_ack_queue = qp->r_head_ack_queue;
243 goto bail;
244 }
245
246 ps->s_txreq->mr = e->rdma_sge.mr;
247 if (ps->s_txreq->mr)
248 rvt_get_mr(ps->s_txreq->mr);
249 qp->s_ack_rdma_sge.sge = e->rdma_sge;
250 qp->s_ack_rdma_sge.num_sge = 1;
251 qp->s_ack_state = TID_OP(READ_RESP);
252 goto read_resp;
253 } else {
254
255 ps->s_txreq->ss = NULL;
256 len = 0;
257 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
258 ohdr->u.at.aeth = rvt_compute_aeth(qp);
259 ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth);
260 hwords += sizeof(ohdr->u.at) / sizeof(u32);
261 bth2 = mask_psn(e->psn);
262 e->sent = 1;
263 }
264 trace_hfi1_tid_write_rsp_make_rc_ack(qp);
265 bth0 = qp->s_ack_state << 24;
266 break;
267
268 case OP(RDMA_READ_RESPONSE_FIRST):
269 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
270
271 case OP(RDMA_READ_RESPONSE_MIDDLE):
272 ps->s_txreq->ss = &qp->s_ack_rdma_sge;
273 ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr;
274 if (ps->s_txreq->mr)
275 rvt_get_mr(ps->s_txreq->mr);
276 len = qp->s_ack_rdma_sge.sge.sge_length;
277 if (len > pmtu) {
278 len = pmtu;
279 middle = HFI1_CAP_IS_KSET(SDMA_AHG);
280 } else {
281 ohdr->u.aeth = rvt_compute_aeth(qp);
282 hwords++;
283 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
284 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
285 e->sent = 1;
286 }
287 bth0 = qp->s_ack_state << 24;
288 bth2 = mask_psn(qp->s_ack_rdma_psn++);
289 break;
290
291 case TID_OP(WRITE_RESP):
292write_resp:
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
313 req = ack_to_tid_req(e);
314
315
316
317
318
319
320 if (qpriv->rnr_nak_state == TID_RNR_NAK_SEND &&
321 qp->s_tail_ack_queue == qpriv->r_tid_alloc &&
322 req->cur_seg == req->alloc_seg) {
323 qpriv->rnr_nak_state = TID_RNR_NAK_SENT;
324 goto normal_no_state;
325 }
326
327 bth2 = mask_psn(qp->s_ack_rdma_psn);
328 hdrlen = hfi1_build_tid_rdma_write_resp(qp, e, ohdr, &bth1,
329 bth2, &len,
330 &ps->s_txreq->ss);
331 if (!hdrlen)
332 return 0;
333
334 hwords += hdrlen;
335 bth0 = qp->s_ack_state << 24;
336 qp->s_ack_rdma_psn++;
337 trace_hfi1_tid_req_make_rc_ack_write(qp, 0, e->opcode, e->psn,
338 e->lpsn, req);
339 if (req->cur_seg != req->total_segs)
340 break;
341
342 e->sent = 1;
343
344 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
345 break;
346
347 case TID_OP(READ_RESP):
348read_resp:
349 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
350 ps->s_txreq->ss = &qp->s_ack_rdma_sge;
351 delta = hfi1_build_tid_rdma_read_resp(qp, e, ohdr, &bth0,
352 &bth1, &bth2, &len,
353 &last_pkt);
354 if (delta == 0)
355 goto error_qp;
356 hwords += delta;
357 if (last_pkt) {
358 e->sent = 1;
359
360
361
362
363 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
364 }
365 break;
366 case TID_OP(READ_REQ):
367 goto bail;
368
369 default:
370normal:
371
372
373
374
375
376
377 qp->s_ack_state = OP(SEND_ONLY);
378normal_no_state:
379 if (qp->s_nak_state)
380 ohdr->u.aeth =
381 cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
382 (qp->s_nak_state <<
383 IB_AETH_CREDIT_SHIFT));
384 else
385 ohdr->u.aeth = rvt_compute_aeth(qp);
386 hwords++;
387 len = 0;
388 bth0 = OP(ACKNOWLEDGE) << 24;
389 bth2 = mask_psn(qp->s_ack_psn);
390 qp->s_flags &= ~RVT_S_ACK_PENDING;
391 ps->s_txreq->txreq.flags |= SDMA_TXREQ_F_VIP;
392 ps->s_txreq->ss = NULL;
393 }
394 qp->s_rdma_ack_cnt++;
395 ps->s_txreq->sde = qpriv->s_sde;
396 ps->s_txreq->s_cur_size = len;
397 ps->s_txreq->hdr_dwords = hwords;
398 hfi1_make_ruc_header(qp, ohdr, bth0, bth1, bth2, middle, ps);
399 return 1;
400error_qp:
401 spin_unlock_irqrestore(&qp->s_lock, ps->flags);
402 spin_lock_irqsave(&qp->r_lock, ps->flags);
403 spin_lock(&qp->s_lock);
404 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
405 spin_unlock(&qp->s_lock);
406 spin_unlock_irqrestore(&qp->r_lock, ps->flags);
407 spin_lock_irqsave(&qp->s_lock, ps->flags);
408bail:
409 qp->s_ack_state = OP(ACKNOWLEDGE);
410
411
412
413
414 smp_wmb();
415 qp->s_flags &= ~(RVT_S_RESP_PENDING
416 | RVT_S_ACK_PENDING
417 | HFI1_S_AHG_VALID);
418 return 0;
419}
420
421
422
423
424
425
426
427
428
429int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
430{
431 struct hfi1_qp_priv *priv = qp->priv;
432 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
433 struct ib_other_headers *ohdr;
434 struct rvt_sge_state *ss = NULL;
435 struct rvt_swqe *wqe;
436 struct hfi1_swqe_priv *wpriv;
437 struct tid_rdma_request *req = NULL;
438
439 u32 hwords = 5;
440 u32 len = 0;
441 u32 bth0 = 0, bth2 = 0;
442 u32 bth1 = qp->remote_qpn | (HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT);
443 u32 pmtu = qp->pmtu;
444 char newreq;
445 int middle = 0;
446 int delta;
447 struct tid_rdma_flow *flow = NULL;
448 struct tid_rdma_params *remote;
449
450 trace_hfi1_sender_make_rc_req(qp);
451 lockdep_assert_held(&qp->s_lock);
452 ps->s_txreq = get_txreq(ps->dev, qp);
453 if (!ps->s_txreq)
454 goto bail_no_tx;
455
456 if (priv->hdr_type == HFI1_PKT_TYPE_9B) {
457
458 hwords = 5;
459 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
460 ohdr = &ps->s_txreq->phdr.hdr.ibh.u.l.oth;
461 else
462 ohdr = &ps->s_txreq->phdr.hdr.ibh.u.oth;
463 } else {
464
465 hwords = 7;
466 if ((rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
467 (hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))))
468 ohdr = &ps->s_txreq->phdr.hdr.opah.u.l.oth;
469 else
470 ohdr = &ps->s_txreq->phdr.hdr.opah.u.oth;
471 }
472
473
474 if ((qp->s_flags & RVT_S_RESP_PENDING) &&
475 make_rc_ack(dev, qp, ohdr, ps))
476 return 1;
477
478 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
479 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
480 goto bail;
481
482 if (qp->s_last == READ_ONCE(qp->s_head))
483 goto bail;
484
485 if (iowait_sdma_pending(&priv->s_iowait)) {
486 qp->s_flags |= RVT_S_WAIT_DMA;
487 goto bail;
488 }
489 clear_ahg(qp);
490 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
491 hfi1_trdma_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
492 IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR);
493
494 goto done_free_tx;
495 }
496
497 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK | HFI1_S_WAIT_HALT))
498 goto bail;
499
500 if (cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) {
501 if (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
502 qp->s_flags |= RVT_S_WAIT_PSN;
503 goto bail;
504 }
505 qp->s_sending_psn = qp->s_psn;
506 qp->s_sending_hpsn = qp->s_psn - 1;
507 }
508
509
510 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
511check_s_state:
512 switch (qp->s_state) {
513 default:
514 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK))
515 goto bail;
516
517
518
519
520
521
522
523 newreq = 0;
524 if (qp->s_cur == qp->s_tail) {
525
526 if (qp->s_tail == READ_ONCE(qp->s_head)) {
527 clear_ahg(qp);
528 goto bail;
529 }
530
531
532
533
534
535
536 if ((wqe->wr.send_flags & IB_SEND_FENCE) &&
537 qp->s_num_rd_atomic &&
538 (wqe->wr.opcode != IB_WR_TID_RDMA_READ ||
539 priv->pending_tid_r_segs < qp->s_num_rd_atomic)) {
540 qp->s_flags |= RVT_S_WAIT_FENCE;
541 goto bail;
542 }
543
544
545
546
547 if (wqe->wr.opcode == IB_WR_REG_MR ||
548 wqe->wr.opcode == IB_WR_LOCAL_INV) {
549 int local_ops = 0;
550 int err = 0;
551
552 if (qp->s_last != qp->s_cur)
553 goto bail;
554 if (++qp->s_cur == qp->s_size)
555 qp->s_cur = 0;
556 if (++qp->s_tail == qp->s_size)
557 qp->s_tail = 0;
558 if (!(wqe->wr.send_flags &
559 RVT_SEND_COMPLETION_ONLY)) {
560 err = rvt_invalidate_rkey(
561 qp,
562 wqe->wr.ex.invalidate_rkey);
563 local_ops = 1;
564 }
565 rvt_send_complete(qp, wqe,
566 err ? IB_WC_LOC_PROT_ERR
567 : IB_WC_SUCCESS);
568 if (local_ops)
569 atomic_dec(&qp->local_ops_pending);
570 goto done_free_tx;
571 }
572
573 newreq = 1;
574 qp->s_psn = wqe->psn;
575 }
576
577
578
579
580
581 len = wqe->length;
582 ss = &qp->s_sge;
583 bth2 = mask_psn(qp->s_psn);
584
585
586
587
588
589 if ((priv->s_flags & HFI1_S_TID_WAIT_INTERLCK) ||
590 hfi1_tid_rdma_wqe_interlock(qp, wqe))
591 goto bail;
592
593 switch (wqe->wr.opcode) {
594 case IB_WR_SEND:
595 case IB_WR_SEND_WITH_IMM:
596 case IB_WR_SEND_WITH_INV:
597
598 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
599 rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
600 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
601 goto bail;
602 }
603 if (len > pmtu) {
604 qp->s_state = OP(SEND_FIRST);
605 len = pmtu;
606 break;
607 }
608 if (wqe->wr.opcode == IB_WR_SEND) {
609 qp->s_state = OP(SEND_ONLY);
610 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
611 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
612
613 ohdr->u.imm_data = wqe->wr.ex.imm_data;
614 hwords += 1;
615 } else {
616 qp->s_state = OP(SEND_ONLY_WITH_INVALIDATE);
617
618 ohdr->u.ieth = cpu_to_be32(
619 wqe->wr.ex.invalidate_rkey);
620 hwords += 1;
621 }
622 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
623 bth0 |= IB_BTH_SOLICITED;
624 bth2 |= IB_BTH_REQ_ACK;
625 if (++qp->s_cur == qp->s_size)
626 qp->s_cur = 0;
627 break;
628
629 case IB_WR_RDMA_WRITE:
630 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
631 qp->s_lsn++;
632 goto no_flow_control;
633 case IB_WR_RDMA_WRITE_WITH_IMM:
634
635 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
636 rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
637 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
638 goto bail;
639 }
640no_flow_control:
641 put_ib_reth_vaddr(
642 wqe->rdma_wr.remote_addr,
643 &ohdr->u.rc.reth);
644 ohdr->u.rc.reth.rkey =
645 cpu_to_be32(wqe->rdma_wr.rkey);
646 ohdr->u.rc.reth.length = cpu_to_be32(len);
647 hwords += sizeof(struct ib_reth) / sizeof(u32);
648 if (len > pmtu) {
649 qp->s_state = OP(RDMA_WRITE_FIRST);
650 len = pmtu;
651 break;
652 }
653 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
654 qp->s_state = OP(RDMA_WRITE_ONLY);
655 } else {
656 qp->s_state =
657 OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
658
659 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;
660 hwords += 1;
661 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
662 bth0 |= IB_BTH_SOLICITED;
663 }
664 bth2 |= IB_BTH_REQ_ACK;
665 if (++qp->s_cur == qp->s_size)
666 qp->s_cur = 0;
667 break;
668
669 case IB_WR_TID_RDMA_WRITE:
670 if (newreq) {
671
672
673
674 if (atomic_read(&priv->n_tid_requests) >=
675 HFI1_TID_RDMA_WRITE_CNT)
676 goto bail;
677
678 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
679 qp->s_lsn++;
680 }
681
682 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr,
683 &bth1, &bth2,
684 &len);
685 ss = NULL;
686 if (priv->s_tid_cur == HFI1_QP_WQE_INVALID) {
687 priv->s_tid_cur = qp->s_cur;
688 if (priv->s_tid_tail == HFI1_QP_WQE_INVALID) {
689 priv->s_tid_tail = qp->s_cur;
690 priv->s_state = TID_OP(WRITE_RESP);
691 }
692 } else if (priv->s_tid_cur == priv->s_tid_head) {
693 struct rvt_swqe *__w;
694 struct tid_rdma_request *__r;
695
696 __w = rvt_get_swqe_ptr(qp, priv->s_tid_cur);
697 __r = wqe_to_tid_req(__w);
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719 if (__w->wr.opcode != IB_WR_TID_RDMA_WRITE ||
720 __r->state == TID_REQUEST_INACTIVE ||
721 __r->state == TID_REQUEST_COMPLETE ||
722 ((__r->state == TID_REQUEST_ACTIVE ||
723 __r->state == TID_REQUEST_SYNC) &&
724 __r->comp_seg == __r->total_segs)) {
725 if (priv->s_tid_tail ==
726 priv->s_tid_cur &&
727 priv->s_state ==
728 TID_OP(WRITE_DATA_LAST)) {
729 priv->s_tid_tail = qp->s_cur;
730 priv->s_state =
731 TID_OP(WRITE_RESP);
732 }
733 priv->s_tid_cur = qp->s_cur;
734 }
735
736
737
738
739
740
741
742
743
744
745 if (priv->s_tid_tail == qp->s_cur &&
746 priv->s_state == TID_OP(WRITE_DATA_LAST))
747 priv->s_state = TID_OP(WRITE_RESP);
748 }
749 req = wqe_to_tid_req(wqe);
750 if (newreq) {
751 priv->s_tid_head = qp->s_cur;
752 priv->pending_tid_w_resp += req->total_segs;
753 atomic_inc(&priv->n_tid_requests);
754 atomic_dec(&priv->n_requests);
755 } else {
756 req->state = TID_REQUEST_RESEND;
757 req->comp_seg = delta_psn(bth2, wqe->psn);
758
759
760
761
762 req->setup_head = req->clear_tail;
763 priv->pending_tid_w_resp +=
764 delta_psn(wqe->lpsn, bth2) + 1;
765 }
766
767 trace_hfi1_tid_write_sender_make_req(qp, newreq);
768 trace_hfi1_tid_req_make_req_write(qp, newreq,
769 wqe->wr.opcode,
770 wqe->psn, wqe->lpsn,
771 req);
772 if (++qp->s_cur == qp->s_size)
773 qp->s_cur = 0;
774 break;
775
776 case IB_WR_RDMA_READ:
777
778
779
780
781 if (qp->s_num_rd_atomic >=
782 qp->s_max_rd_atomic) {
783 qp->s_flags |= RVT_S_WAIT_RDMAR;
784 goto bail;
785 }
786 qp->s_num_rd_atomic++;
787 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
788 qp->s_lsn++;
789 put_ib_reth_vaddr(
790 wqe->rdma_wr.remote_addr,
791 &ohdr->u.rc.reth);
792 ohdr->u.rc.reth.rkey =
793 cpu_to_be32(wqe->rdma_wr.rkey);
794 ohdr->u.rc.reth.length = cpu_to_be32(len);
795 qp->s_state = OP(RDMA_READ_REQUEST);
796 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
797 ss = NULL;
798 len = 0;
799 bth2 |= IB_BTH_REQ_ACK;
800 if (++qp->s_cur == qp->s_size)
801 qp->s_cur = 0;
802 break;
803
804 case IB_WR_TID_RDMA_READ:
805 trace_hfi1_tid_read_sender_make_req(qp, newreq);
806 wpriv = wqe->priv;
807 req = wqe_to_tid_req(wqe);
808 trace_hfi1_tid_req_make_req_read(qp, newreq,
809 wqe->wr.opcode,
810 wqe->psn, wqe->lpsn,
811 req);
812 delta = cmp_psn(qp->s_psn, wqe->psn);
813
814
815
816
817
818
819
820
821
822
823
824 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) {
825 qp->s_flags |= RVT_S_WAIT_RDMAR;
826 goto bail;
827 }
828 if (newreq) {
829 struct tid_rdma_flow *flow =
830 &req->flows[req->setup_head];
831
832
833
834
835
836
837
838 if (!flow->npagesets) {
839 qp->s_sge.sge = wqe->sg_list[0];
840 qp->s_sge.sg_list = wqe->sg_list + 1;
841 qp->s_sge.num_sge = wqe->wr.num_sge;
842 qp->s_sge.total_len = wqe->length;
843 qp->s_len = wqe->length;
844 req->isge = 0;
845 req->clear_tail = req->setup_head;
846 req->flow_idx = req->setup_head;
847 req->state = TID_REQUEST_ACTIVE;
848 }
849 } else if (delta == 0) {
850
851 req->cur_seg = 0;
852 req->comp_seg = 0;
853 req->ack_pending = 0;
854 req->flow_idx = req->clear_tail;
855 req->state = TID_REQUEST_RESEND;
856 }
857 req->s_next_psn = qp->s_psn;
858
859 len = min_t(u32, req->seg_len,
860 wqe->length - req->seg_len * req->cur_seg);
861 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr,
862 &bth1, &bth2,
863 &len);
864 if (delta <= 0) {
865
866 goto bail;
867 }
868 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
869 qp->s_lsn++;
870 hwords += delta;
871 ss = &wpriv->ss;
872
873 if (req->cur_seg >= req->total_segs &&
874 ++qp->s_cur == qp->s_size)
875 qp->s_cur = 0;
876 break;
877
878 case IB_WR_ATOMIC_CMP_AND_SWP:
879 case IB_WR_ATOMIC_FETCH_AND_ADD:
880
881
882
883
884 if (qp->s_num_rd_atomic >=
885 qp->s_max_rd_atomic) {
886 qp->s_flags |= RVT_S_WAIT_RDMAR;
887 goto bail;
888 }
889 qp->s_num_rd_atomic++;
890
891
892 case IB_WR_OPFN:
893 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
894 qp->s_lsn++;
895 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
896 wqe->wr.opcode == IB_WR_OPFN) {
897 qp->s_state = OP(COMPARE_SWAP);
898 put_ib_ateth_swap(wqe->atomic_wr.swap,
899 &ohdr->u.atomic_eth);
900 put_ib_ateth_compare(wqe->atomic_wr.compare_add,
901 &ohdr->u.atomic_eth);
902 } else {
903 qp->s_state = OP(FETCH_ADD);
904 put_ib_ateth_swap(wqe->atomic_wr.compare_add,
905 &ohdr->u.atomic_eth);
906 put_ib_ateth_compare(0, &ohdr->u.atomic_eth);
907 }
908 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
909 &ohdr->u.atomic_eth);
910 ohdr->u.atomic_eth.rkey = cpu_to_be32(
911 wqe->atomic_wr.rkey);
912 hwords += sizeof(struct ib_atomic_eth) / sizeof(u32);
913 ss = NULL;
914 len = 0;
915 bth2 |= IB_BTH_REQ_ACK;
916 if (++qp->s_cur == qp->s_size)
917 qp->s_cur = 0;
918 break;
919
920 default:
921 goto bail;
922 }
923 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ) {
924 qp->s_sge.sge = wqe->sg_list[0];
925 qp->s_sge.sg_list = wqe->sg_list + 1;
926 qp->s_sge.num_sge = wqe->wr.num_sge;
927 qp->s_sge.total_len = wqe->length;
928 qp->s_len = wqe->length;
929 }
930 if (newreq) {
931 qp->s_tail++;
932 if (qp->s_tail >= qp->s_size)
933 qp->s_tail = 0;
934 }
935 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
936 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
937 qp->s_psn = wqe->lpsn + 1;
938 else if (wqe->wr.opcode == IB_WR_TID_RDMA_READ)
939 qp->s_psn = req->s_next_psn;
940 else
941 qp->s_psn++;
942 break;
943
944 case OP(RDMA_READ_RESPONSE_FIRST):
945
946
947
948
949
950
951
952
953
954 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
955
956 case OP(SEND_FIRST):
957 qp->s_state = OP(SEND_MIDDLE);
958
959 case OP(SEND_MIDDLE):
960 bth2 = mask_psn(qp->s_psn++);
961 ss = &qp->s_sge;
962 len = qp->s_len;
963 if (len > pmtu) {
964 len = pmtu;
965 middle = HFI1_CAP_IS_KSET(SDMA_AHG);
966 break;
967 }
968 if (wqe->wr.opcode == IB_WR_SEND) {
969 qp->s_state = OP(SEND_LAST);
970 } else if (wqe->wr.opcode == IB_WR_SEND_WITH_IMM) {
971 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
972
973 ohdr->u.imm_data = wqe->wr.ex.imm_data;
974 hwords += 1;
975 } else {
976 qp->s_state = OP(SEND_LAST_WITH_INVALIDATE);
977
978 ohdr->u.ieth = cpu_to_be32(wqe->wr.ex.invalidate_rkey);
979 hwords += 1;
980 }
981 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
982 bth0 |= IB_BTH_SOLICITED;
983 bth2 |= IB_BTH_REQ_ACK;
984 qp->s_cur++;
985 if (qp->s_cur >= qp->s_size)
986 qp->s_cur = 0;
987 break;
988
989 case OP(RDMA_READ_RESPONSE_LAST):
990
991
992
993
994
995
996
997
998
999 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
1000
1001 case OP(RDMA_WRITE_FIRST):
1002 qp->s_state = OP(RDMA_WRITE_MIDDLE);
1003
1004 case OP(RDMA_WRITE_MIDDLE):
1005 bth2 = mask_psn(qp->s_psn++);
1006 ss = &qp->s_sge;
1007 len = qp->s_len;
1008 if (len > pmtu) {
1009 len = pmtu;
1010 middle = HFI1_CAP_IS_KSET(SDMA_AHG);
1011 break;
1012 }
1013 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
1014 qp->s_state = OP(RDMA_WRITE_LAST);
1015 } else {
1016 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
1017
1018 ohdr->u.imm_data = wqe->wr.ex.imm_data;
1019 hwords += 1;
1020 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
1021 bth0 |= IB_BTH_SOLICITED;
1022 }
1023 bth2 |= IB_BTH_REQ_ACK;
1024 qp->s_cur++;
1025 if (qp->s_cur >= qp->s_size)
1026 qp->s_cur = 0;
1027 break;
1028
1029 case OP(RDMA_READ_RESPONSE_MIDDLE):
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu;
1040 put_ib_reth_vaddr(
1041 wqe->rdma_wr.remote_addr + len,
1042 &ohdr->u.rc.reth);
1043 ohdr->u.rc.reth.rkey =
1044 cpu_to_be32(wqe->rdma_wr.rkey);
1045 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
1046 qp->s_state = OP(RDMA_READ_REQUEST);
1047 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
1048 bth2 = mask_psn(qp->s_psn) | IB_BTH_REQ_ACK;
1049 qp->s_psn = wqe->lpsn + 1;
1050 ss = NULL;
1051 len = 0;
1052 qp->s_cur++;
1053 if (qp->s_cur == qp->s_size)
1054 qp->s_cur = 0;
1055 break;
1056
1057 case TID_OP(WRITE_RESP):
1058
1059
1060
1061
1062
1063 req = wqe_to_tid_req(wqe);
1064 req->state = TID_REQUEST_RESEND;
1065 rcu_read_lock();
1066 remote = rcu_dereference(priv->tid_rdma.remote);
1067 req->comp_seg = delta_psn(qp->s_psn, wqe->psn);
1068 len = wqe->length - (req->comp_seg * remote->max_len);
1069 rcu_read_unlock();
1070
1071 bth2 = mask_psn(qp->s_psn);
1072 hwords += hfi1_build_tid_rdma_write_req(qp, wqe, ohdr, &bth1,
1073 &bth2, &len);
1074 qp->s_psn = wqe->lpsn + 1;
1075 ss = NULL;
1076 qp->s_state = TID_OP(WRITE_REQ);
1077 priv->pending_tid_w_resp += delta_psn(wqe->lpsn, bth2) + 1;
1078 priv->s_tid_cur = qp->s_cur;
1079 if (++qp->s_cur == qp->s_size)
1080 qp->s_cur = 0;
1081 trace_hfi1_tid_req_make_req_write(qp, 0, wqe->wr.opcode,
1082 wqe->psn, wqe->lpsn, req);
1083 break;
1084
1085 case TID_OP(READ_RESP):
1086 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ)
1087 goto bail;
1088
1089 req = wqe_to_tid_req(wqe);
1090 wpriv = wqe->priv;
1091
1092
1093
1094
1095
1096 req->cur_seg = delta_psn(qp->s_psn, wqe->psn) / priv->pkts_ps;
1097
1098
1099
1100
1101
1102
1103
1104 req->state = TID_REQUEST_RESEND;
1105 hfi1_tid_rdma_restart_req(qp, wqe, &bth2);
1106 if (req->state != TID_REQUEST_ACTIVE) {
1107
1108
1109
1110
1111 hfi1_kern_exp_rcv_clear_all(req);
1112 hfi1_kern_clear_hw_flow(priv->rcd, qp);
1113
1114 hfi1_trdma_send_complete(qp, wqe, IB_WC_LOC_QP_OP_ERR);
1115 goto bail;
1116 }
1117 req->state = TID_REQUEST_RESEND;
1118 len = min_t(u32, req->seg_len,
1119 wqe->length - req->seg_len * req->cur_seg);
1120 flow = &req->flows[req->flow_idx];
1121 len -= flow->sent;
1122 req->s_next_psn = flow->flow_state.ib_lpsn + 1;
1123 delta = hfi1_build_tid_rdma_read_packet(wqe, ohdr, &bth1,
1124 &bth2, &len);
1125 if (delta <= 0) {
1126
1127 goto bail;
1128 }
1129 hwords += delta;
1130 ss = &wpriv->ss;
1131
1132 if (req->cur_seg >= req->total_segs &&
1133 ++qp->s_cur == qp->s_size)
1134 qp->s_cur = 0;
1135 qp->s_psn = req->s_next_psn;
1136 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode,
1137 wqe->psn, wqe->lpsn, req);
1138 break;
1139 case TID_OP(READ_REQ):
1140 req = wqe_to_tid_req(wqe);
1141 delta = cmp_psn(qp->s_psn, wqe->psn);
1142
1143
1144
1145
1146
1147 if (wqe->wr.opcode != IB_WR_TID_RDMA_READ || delta == 0 ||
1148 qp->s_cur == qp->s_tail) {
1149 qp->s_state = OP(RDMA_READ_REQUEST);
1150 if (delta == 0 || qp->s_cur == qp->s_tail)
1151 goto check_s_state;
1152 else
1153 goto bail;
1154 }
1155
1156
1157 if (qp->s_num_rd_atomic >= qp->s_max_rd_atomic) {
1158 qp->s_flags |= RVT_S_WAIT_RDMAR;
1159 goto bail;
1160 }
1161
1162 wpriv = wqe->priv;
1163
1164 len = min_t(u32, req->seg_len,
1165 wqe->length - req->seg_len * req->cur_seg);
1166 delta = hfi1_build_tid_rdma_read_req(qp, wqe, ohdr, &bth1,
1167 &bth2, &len);
1168 if (delta <= 0) {
1169
1170 goto bail;
1171 }
1172 hwords += delta;
1173 ss = &wpriv->ss;
1174
1175 if (req->cur_seg >= req->total_segs &&
1176 ++qp->s_cur == qp->s_size)
1177 qp->s_cur = 0;
1178 qp->s_psn = req->s_next_psn;
1179 trace_hfi1_tid_req_make_req_read(qp, 0, wqe->wr.opcode,
1180 wqe->psn, wqe->lpsn, req);
1181 break;
1182 }
1183 qp->s_sending_hpsn = bth2;
1184 delta = delta_psn(bth2, wqe->psn);
1185 if (delta && delta % HFI1_PSN_CREDIT == 0 &&
1186 wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)
1187 bth2 |= IB_BTH_REQ_ACK;
1188 if (qp->s_flags & RVT_S_SEND_ONE) {
1189 qp->s_flags &= ~RVT_S_SEND_ONE;
1190 qp->s_flags |= RVT_S_WAIT_ACK;
1191 bth2 |= IB_BTH_REQ_ACK;
1192 }
1193 qp->s_len -= len;
1194 ps->s_txreq->hdr_dwords = hwords;
1195 ps->s_txreq->sde = priv->s_sde;
1196 ps->s_txreq->ss = ss;
1197 ps->s_txreq->s_cur_size = len;
1198 hfi1_make_ruc_header(
1199 qp,
1200 ohdr,
1201 bth0 | (qp->s_state << 24),
1202 bth1,
1203 bth2,
1204 middle,
1205 ps);
1206 return 1;
1207
1208done_free_tx:
1209 hfi1_put_txreq(ps->s_txreq);
1210 ps->s_txreq = NULL;
1211 return 1;
1212
1213bail:
1214 hfi1_put_txreq(ps->s_txreq);
1215
1216bail_no_tx:
1217 ps->s_txreq = NULL;
1218 qp->s_flags &= ~RVT_S_BUSY;
1219
1220
1221
1222
1223
1224 iowait_set_flag(&priv->s_iowait, IOWAIT_PENDING_IB);
1225 return 0;
1226}
1227
1228static inline void hfi1_make_bth_aeth(struct rvt_qp *qp,
1229 struct ib_other_headers *ohdr,
1230 u32 bth0, u32 bth1)
1231{
1232 if (qp->r_nak_state)
1233 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
1234 (qp->r_nak_state <<
1235 IB_AETH_CREDIT_SHIFT));
1236 else
1237 ohdr->u.aeth = rvt_compute_aeth(qp);
1238
1239 ohdr->bth[0] = cpu_to_be32(bth0);
1240 ohdr->bth[1] = cpu_to_be32(bth1 | qp->remote_qpn);
1241 ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn));
1242}
1243
1244static inline void hfi1_queue_rc_ack(struct hfi1_packet *packet, bool is_fecn)
1245{
1246 struct rvt_qp *qp = packet->qp;
1247 struct hfi1_ibport *ibp;
1248 unsigned long flags;
1249
1250 spin_lock_irqsave(&qp->s_lock, flags);
1251 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
1252 goto unlock;
1253 ibp = rcd_to_iport(packet->rcd);
1254 this_cpu_inc(*ibp->rvp.rc_qacks);
1255 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
1256 qp->s_nak_state = qp->r_nak_state;
1257 qp->s_ack_psn = qp->r_ack_psn;
1258 if (is_fecn)
1259 qp->s_flags |= RVT_S_ECN;
1260
1261
1262 hfi1_schedule_send(qp);
1263unlock:
1264 spin_unlock_irqrestore(&qp->s_lock, flags);
1265}
1266
1267static inline void hfi1_make_rc_ack_9B(struct hfi1_packet *packet,
1268 struct hfi1_opa_header *opa_hdr,
1269 u8 sc5, bool is_fecn,
1270 u64 *pbc_flags, u32 *hwords,
1271 u32 *nwords)
1272{
1273 struct rvt_qp *qp = packet->qp;
1274 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
1275 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
1276 struct ib_header *hdr = &opa_hdr->ibh;
1277 struct ib_other_headers *ohdr;
1278 u16 lrh0 = HFI1_LRH_BTH;
1279 u16 pkey;
1280 u32 bth0, bth1;
1281
1282 opa_hdr->hdr_type = HFI1_PKT_TYPE_9B;
1283 ohdr = &hdr->u.oth;
1284
1285 *hwords = 6;
1286
1287 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)) {
1288 *hwords += hfi1_make_grh(ibp, &hdr->u.l.grh,
1289 rdma_ah_read_grh(&qp->remote_ah_attr),
1290 *hwords - 2, SIZE_OF_CRC);
1291 ohdr = &hdr->u.l.oth;
1292 lrh0 = HFI1_LRH_GRH;
1293 }
1294
1295 *pbc_flags |= ((!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT);
1296
1297
1298 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
1299
1300 lrh0 |= (sc5 & IB_SC_MASK) << IB_SC_SHIFT |
1301 (rdma_ah_get_sl(&qp->remote_ah_attr) & IB_SL_MASK) <<
1302 IB_SL_SHIFT;
1303
1304 hfi1_make_ib_hdr(hdr, lrh0, *hwords + SIZE_OF_CRC,
1305 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr), 9B),
1306 ppd->lid | rdma_ah_get_path_bits(&qp->remote_ah_attr));
1307
1308 bth0 = pkey | (OP(ACKNOWLEDGE) << 24);
1309 if (qp->s_mig_state == IB_MIG_MIGRATED)
1310 bth0 |= IB_BTH_MIG_REQ;
1311 bth1 = (!!is_fecn) << IB_BECN_SHIFT;
1312
1313
1314
1315
1316 bth1 |= HFI1_CAP_IS_KSET(OPFN) << IB_BTHE_E_SHIFT;
1317 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
1318}
1319
1320static inline void hfi1_make_rc_ack_16B(struct hfi1_packet *packet,
1321 struct hfi1_opa_header *opa_hdr,
1322 u8 sc5, bool is_fecn,
1323 u64 *pbc_flags, u32 *hwords,
1324 u32 *nwords)
1325{
1326 struct rvt_qp *qp = packet->qp;
1327 struct hfi1_ibport *ibp = rcd_to_iport(packet->rcd);
1328 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
1329 struct hfi1_16b_header *hdr = &opa_hdr->opah;
1330 struct ib_other_headers *ohdr;
1331 u32 bth0, bth1 = 0;
1332 u16 len, pkey;
1333 bool becn = is_fecn;
1334 u8 l4 = OPA_16B_L4_IB_LOCAL;
1335 u8 extra_bytes;
1336
1337 opa_hdr->hdr_type = HFI1_PKT_TYPE_16B;
1338 ohdr = &hdr->u.oth;
1339
1340 *hwords = 8;
1341 extra_bytes = hfi1_get_16b_padding(*hwords << 2, 0);
1342 *nwords = SIZE_OF_CRC + ((extra_bytes + SIZE_OF_LT) >> 2);
1343
1344 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) &&
1345 hfi1_check_mcast(rdma_ah_get_dlid(&qp->remote_ah_attr))) {
1346 *hwords += hfi1_make_grh(ibp, &hdr->u.l.grh,
1347 rdma_ah_read_grh(&qp->remote_ah_attr),
1348 *hwords - 4, *nwords);
1349 ohdr = &hdr->u.l.oth;
1350 l4 = OPA_16B_L4_IB_GLOBAL;
1351 }
1352 *pbc_flags |= PBC_PACKET_BYPASS | PBC_INSERT_BYPASS_ICRC;
1353
1354
1355 pkey = hfi1_get_pkey(ibp, qp->s_pkey_index);
1356
1357
1358 len = (*hwords + *nwords) >> 1;
1359
1360 hfi1_make_16b_hdr(hdr, ppd->lid |
1361 (rdma_ah_get_path_bits(&qp->remote_ah_attr) &
1362 ((1 << ppd->lmc) - 1)),
1363 opa_get_lid(rdma_ah_get_dlid(&qp->remote_ah_attr),
1364 16B), len, pkey, becn, 0, l4, sc5);
1365
1366 bth0 = pkey | (OP(ACKNOWLEDGE) << 24);
1367 bth0 |= extra_bytes << 20;
1368 if (qp->s_mig_state == IB_MIG_MIGRATED)
1369 bth1 = OPA_BTH_MIG_REQ;
1370 hfi1_make_bth_aeth(qp, ohdr, bth0, bth1);
1371}
1372
1373typedef void (*hfi1_make_rc_ack)(struct hfi1_packet *packet,
1374 struct hfi1_opa_header *opa_hdr,
1375 u8 sc5, bool is_fecn,
1376 u64 *pbc_flags, u32 *hwords,
1377 u32 *nwords);
1378
1379
1380static const hfi1_make_rc_ack hfi1_make_rc_ack_tbl[2] = {
1381 [HFI1_PKT_TYPE_9B] = &hfi1_make_rc_ack_9B,
1382 [HFI1_PKT_TYPE_16B] = &hfi1_make_rc_ack_16B
1383};
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393void hfi1_send_rc_ack(struct hfi1_packet *packet, bool is_fecn)
1394{
1395 struct hfi1_ctxtdata *rcd = packet->rcd;
1396 struct rvt_qp *qp = packet->qp;
1397 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
1398 struct hfi1_qp_priv *priv = qp->priv;
1399 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
1400 u8 sc5 = ibp->sl_to_sc[rdma_ah_get_sl(&qp->remote_ah_attr)];
1401 u64 pbc, pbc_flags = 0;
1402 u32 hwords = 0;
1403 u32 nwords = 0;
1404 u32 plen;
1405 struct pio_buf *pbuf;
1406 struct hfi1_opa_header opa_hdr;
1407
1408
1409 qp->r_adefered = 0;
1410
1411
1412 if (qp->s_flags & RVT_S_RESP_PENDING) {
1413 hfi1_queue_rc_ack(packet, is_fecn);
1414 return;
1415 }
1416
1417
1418 if (qp->s_rdma_ack_cnt) {
1419 hfi1_queue_rc_ack(packet, is_fecn);
1420 return;
1421 }
1422
1423
1424 if (driver_lstate(ppd) != IB_PORT_ACTIVE)
1425 return;
1426
1427
1428 hfi1_make_rc_ack_tbl[priv->hdr_type](packet, &opa_hdr, sc5, is_fecn,
1429 &pbc_flags, &hwords, &nwords);
1430
1431 plen = 2 + hwords + nwords;
1432 pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps,
1433 sc_to_vlt(ppd->dd, sc5), plen);
1434 pbuf = sc_buffer_alloc(rcd->sc, plen, NULL, NULL);
1435 if (IS_ERR_OR_NULL(pbuf)) {
1436
1437
1438
1439
1440
1441
1442 hfi1_queue_rc_ack(packet, is_fecn);
1443 return;
1444 }
1445 trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device),
1446 &opa_hdr, ib_is_sc5(sc5));
1447
1448
1449 ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc,
1450 (priv->hdr_type == HFI1_PKT_TYPE_9B ?
1451 (void *)&opa_hdr.ibh :
1452 (void *)&opa_hdr.opah), hwords);
1453 return;
1454}
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466static void update_num_rd_atomic(struct rvt_qp *qp, u32 psn,
1467 struct rvt_swqe *wqe)
1468{
1469 u32 opcode = wqe->wr.opcode;
1470
1471 if (opcode == IB_WR_RDMA_READ ||
1472 opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1473 opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
1474 qp->s_num_rd_atomic++;
1475 } else if (opcode == IB_WR_TID_RDMA_READ) {
1476 struct tid_rdma_request *req = wqe_to_tid_req(wqe);
1477 struct hfi1_qp_priv *priv = qp->priv;
1478
1479 if (cmp_psn(psn, wqe->lpsn) <= 0) {
1480 u32 cur_seg;
1481
1482 cur_seg = (psn - wqe->psn) / priv->pkts_ps;
1483 req->ack_pending = cur_seg - req->comp_seg;
1484 priv->pending_tid_r_segs += req->ack_pending;
1485 qp->s_num_rd_atomic += req->ack_pending;
1486 } else {
1487 priv->pending_tid_r_segs += req->total_segs;
1488 qp->s_num_rd_atomic += req->total_segs;
1489 }
1490 }
1491}
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502static void reset_psn(struct rvt_qp *qp, u32 psn)
1503{
1504 u32 n = qp->s_acked;
1505 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
1506 u32 opcode;
1507 struct hfi1_qp_priv *priv = qp->priv;
1508
1509 lockdep_assert_held(&qp->s_lock);
1510 qp->s_cur = n;
1511 priv->pending_tid_r_segs = 0;
1512 priv->pending_tid_w_resp = 0;
1513 qp->s_num_rd_atomic = 0;
1514
1515
1516
1517
1518
1519 if (cmp_psn(psn, wqe->psn) <= 0) {
1520 qp->s_state = OP(SEND_LAST);
1521 goto done;
1522 }
1523 update_num_rd_atomic(qp, psn, wqe);
1524
1525
1526 for (;;) {
1527 int diff;
1528
1529 if (++n == qp->s_size)
1530 n = 0;
1531 if (n == qp->s_tail)
1532 break;
1533 wqe = rvt_get_swqe_ptr(qp, n);
1534 diff = cmp_psn(psn, wqe->psn);
1535 if (diff < 0) {
1536
1537 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1538 break;
1539 }
1540 qp->s_cur = n;
1541
1542
1543
1544
1545 if (diff == 0) {
1546 qp->s_state = OP(SEND_LAST);
1547 goto done;
1548 }
1549
1550 update_num_rd_atomic(qp, psn, wqe);
1551 }
1552 opcode = wqe->wr.opcode;
1553
1554
1555
1556
1557
1558
1559 switch (opcode) {
1560 case IB_WR_SEND:
1561 case IB_WR_SEND_WITH_IMM:
1562 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
1563 break;
1564
1565 case IB_WR_RDMA_WRITE:
1566 case IB_WR_RDMA_WRITE_WITH_IMM:
1567 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
1568 break;
1569
1570 case IB_WR_TID_RDMA_WRITE:
1571 qp->s_state = TID_OP(WRITE_RESP);
1572 break;
1573
1574 case IB_WR_RDMA_READ:
1575 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
1576 break;
1577
1578 case IB_WR_TID_RDMA_READ:
1579 qp->s_state = TID_OP(READ_RESP);
1580 break;
1581
1582 default:
1583
1584
1585
1586
1587 qp->s_state = OP(SEND_LAST);
1588 }
1589done:
1590 priv->s_flags &= ~HFI1_S_TID_WAIT_INTERLCK;
1591 qp->s_psn = psn;
1592
1593
1594
1595
1596
1597 if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
1598 (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
1599 qp->s_flags |= RVT_S_WAIT_PSN;
1600 qp->s_flags &= ~HFI1_S_AHG_VALID;
1601 trace_hfi1_sender_reset_psn(qp);
1602}
1603
1604
1605
1606
1607
1608void hfi1_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
1609{
1610 struct hfi1_qp_priv *priv = qp->priv;
1611 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1612 struct hfi1_ibport *ibp;
1613
1614 lockdep_assert_held(&qp->r_lock);
1615 lockdep_assert_held(&qp->s_lock);
1616 trace_hfi1_sender_restart_rc(qp);
1617 if (qp->s_retry == 0) {
1618 if (qp->s_mig_state == IB_MIG_ARMED) {
1619 hfi1_migrate_qp(qp);
1620 qp->s_retry = qp->s_retry_cnt;
1621 } else if (qp->s_last == qp->s_acked) {
1622
1623
1624
1625
1626 if (wqe->wr.opcode == IB_WR_OPFN) {
1627 struct hfi1_ibport *ibp =
1628 to_iport(qp->ibqp.device, qp->port_num);
1629
1630
1631
1632
1633
1634 opfn_conn_reply(qp, priv->opfn.curr);
1635 wqe = do_rc_completion(qp, wqe, ibp);
1636 qp->s_flags &= ~RVT_S_WAIT_ACK;
1637 } else {
1638 trace_hfi1_tid_write_sender_restart_rc(qp, 0);
1639 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
1640 struct tid_rdma_request *req;
1641
1642 req = wqe_to_tid_req(wqe);
1643 hfi1_kern_exp_rcv_clear_all(req);
1644 hfi1_kern_clear_hw_flow(priv->rcd, qp);
1645 }
1646
1647 hfi1_trdma_send_complete(qp, wqe,
1648 IB_WC_RETRY_EXC_ERR);
1649 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1650 }
1651 return;
1652 } else {
1653 return;
1654 }
1655 } else {
1656 qp->s_retry--;
1657 }
1658
1659 ibp = to_iport(qp->ibqp.device, qp->port_num);
1660 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1661 wqe->wr.opcode == IB_WR_TID_RDMA_READ)
1662 ibp->rvp.n_rc_resends++;
1663 else
1664 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn);
1665
1666 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR |
1667 RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_PSN |
1668 RVT_S_WAIT_ACK | HFI1_S_WAIT_TID_RESP);
1669 if (wait)
1670 qp->s_flags |= RVT_S_SEND_ONE;
1671 reset_psn(qp, psn);
1672}
1673
1674
1675
1676
1677
1678
1679static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
1680{
1681 struct rvt_swqe *wqe;
1682 u32 n = qp->s_last;
1683
1684 lockdep_assert_held(&qp->s_lock);
1685
1686 for (;;) {
1687 wqe = rvt_get_swqe_ptr(qp, n);
1688 if (cmp_psn(psn, wqe->lpsn) <= 0) {
1689 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1690 wqe->wr.opcode == IB_WR_TID_RDMA_READ ||
1691 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
1692 qp->s_sending_psn = wqe->lpsn + 1;
1693 else
1694 qp->s_sending_psn = psn + 1;
1695 break;
1696 }
1697 if (++n == qp->s_size)
1698 n = 0;
1699 if (n == qp->s_tail)
1700 break;
1701 }
1702}
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716void hfi1_rc_verbs_aborted(struct rvt_qp *qp, struct hfi1_opa_header *opah)
1717{
1718 struct ib_other_headers *ohdr = hfi1_get_rc_ohdr(opah);
1719 u8 opcode = ib_bth_get_opcode(ohdr);
1720 u32 psn;
1721
1722
1723 if ((opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
1724 opcode <= OP(ATOMIC_ACKNOWLEDGE)) ||
1725 opcode == TID_OP(READ_RESP) ||
1726 opcode == TID_OP(WRITE_RESP))
1727 return;
1728
1729 psn = ib_bth_get_psn(ohdr) | IB_BTH_REQ_ACK;
1730 ohdr->bth[2] = cpu_to_be32(psn);
1731 qp->s_flags |= RVT_S_SEND_ONE;
1732}
1733
1734
1735
1736
1737void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah)
1738{
1739 struct ib_other_headers *ohdr;
1740 struct hfi1_qp_priv *priv = qp->priv;
1741 struct rvt_swqe *wqe;
1742 u32 opcode, head, tail;
1743 u32 psn;
1744 struct tid_rdma_request *req;
1745
1746 lockdep_assert_held(&qp->s_lock);
1747 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK))
1748 return;
1749
1750 ohdr = hfi1_get_rc_ohdr(opah);
1751 opcode = ib_bth_get_opcode(ohdr);
1752 if ((opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
1753 opcode <= OP(ATOMIC_ACKNOWLEDGE)) ||
1754 opcode == TID_OP(READ_RESP) ||
1755 opcode == TID_OP(WRITE_RESP)) {
1756 WARN_ON(!qp->s_rdma_ack_cnt);
1757 qp->s_rdma_ack_cnt--;
1758 return;
1759 }
1760
1761 psn = ib_bth_get_psn(ohdr);
1762
1763
1764
1765
1766 if (opcode != TID_OP(WRITE_DATA) &&
1767 opcode != TID_OP(WRITE_DATA_LAST) &&
1768 opcode != TID_OP(ACK) && opcode != TID_OP(RESYNC))
1769 reset_sending_psn(qp, psn);
1770
1771
1772 if (opcode >= TID_OP(WRITE_REQ) &&
1773 opcode <= TID_OP(WRITE_DATA_LAST)) {
1774 head = priv->s_tid_head;
1775 tail = priv->s_tid_cur;
1776
1777
1778
1779
1780
1781
1782
1783
1784 wqe = rvt_get_swqe_ptr(qp, tail);
1785 req = wqe_to_tid_req(wqe);
1786 if (head == tail && req->comp_seg < req->total_segs) {
1787 if (tail == 0)
1788 tail = qp->s_size - 1;
1789 else
1790 tail -= 1;
1791 }
1792 } else {
1793 head = qp->s_tail;
1794 tail = qp->s_acked;
1795 }
1796
1797
1798
1799
1800
1801 if ((psn & IB_BTH_REQ_ACK) && tail != head &&
1802 opcode != TID_OP(WRITE_DATA) && opcode != TID_OP(WRITE_DATA_LAST) &&
1803 opcode != TID_OP(RESYNC) &&
1804 !(qp->s_flags &
1805 (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) &&
1806 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
1807 if (opcode == TID_OP(READ_REQ))
1808 rvt_add_retry_timer_ext(qp, priv->timeout_shift);
1809 else
1810 rvt_add_retry_timer(qp);
1811 }
1812
1813
1814 if ((opcode == TID_OP(WRITE_DATA) ||
1815 opcode == TID_OP(WRITE_DATA_LAST) ||
1816 opcode == TID_OP(RESYNC)) &&
1817 (psn & IB_BTH_REQ_ACK) &&
1818 !(priv->s_flags & HFI1_S_TID_RETRY_TIMER) &&
1819 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) {
1820
1821
1822
1823
1824
1825 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1826 req = wqe_to_tid_req(wqe);
1827 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE &&
1828 req->ack_seg < req->cur_seg)
1829 hfi1_add_tid_retry_timer(qp);
1830 }
1831
1832 while (qp->s_last != qp->s_acked) {
1833 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
1834 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 &&
1835 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
1836 break;
1837 trdma_clean_swqe(qp, wqe);
1838 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
1839 rvt_qp_complete_swqe(qp,
1840 wqe,
1841 ib_hfi1_wc_opcode[wqe->wr.opcode],
1842 IB_WC_SUCCESS);
1843 }
1844
1845
1846
1847
1848 trace_hfi1_sendcomplete(qp, psn);
1849 if (qp->s_flags & RVT_S_WAIT_PSN &&
1850 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1851 qp->s_flags &= ~RVT_S_WAIT_PSN;
1852 qp->s_sending_psn = qp->s_psn;
1853 qp->s_sending_hpsn = qp->s_psn - 1;
1854 hfi1_schedule_send(qp);
1855 }
1856}
1857
1858static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
1859{
1860 qp->s_last_psn = psn;
1861}
1862
1863
1864
1865
1866
1867
1868struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
1869 struct rvt_swqe *wqe,
1870 struct hfi1_ibport *ibp)
1871{
1872 struct hfi1_qp_priv *priv = qp->priv;
1873
1874 lockdep_assert_held(&qp->s_lock);
1875
1876
1877
1878
1879
1880 trace_hfi1_rc_completion(qp, wqe->lpsn);
1881 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 ||
1882 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1883 trdma_clean_swqe(qp, wqe);
1884 trace_hfi1_qp_send_completion(qp, wqe, qp->s_last);
1885 rvt_qp_complete_swqe(qp,
1886 wqe,
1887 ib_hfi1_wc_opcode[wqe->wr.opcode],
1888 IB_WC_SUCCESS);
1889 } else {
1890 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
1891
1892 this_cpu_inc(*ibp->rvp.rc_delayed_comp);
1893
1894
1895
1896
1897 if (ppd->dd->flags & HFI1_HAS_SEND_DMA) {
1898 struct sdma_engine *engine;
1899 u8 sl = rdma_ah_get_sl(&qp->remote_ah_attr);
1900 u8 sc5;
1901
1902
1903 sc5 = ibp->sl_to_sc[sl];
1904 engine = qp_to_sdma_engine(qp, sc5);
1905 sdma_engine_progress_schedule(engine);
1906 }
1907 }
1908
1909 qp->s_retry = qp->s_retry_cnt;
1910
1911
1912
1913
1914
1915
1916
1917
1918 if (wqe->wr.opcode != IB_WR_TID_RDMA_WRITE)
1919 update_last_psn(qp, wqe->lpsn);
1920
1921
1922
1923
1924
1925
1926 if (qp->s_acked == qp->s_cur) {
1927 if (++qp->s_cur >= qp->s_size)
1928 qp->s_cur = 0;
1929 qp->s_acked = qp->s_cur;
1930 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1931 if (qp->s_acked != qp->s_tail) {
1932 qp->s_state = OP(SEND_LAST);
1933 qp->s_psn = wqe->psn;
1934 }
1935 } else {
1936 if (++qp->s_acked >= qp->s_size)
1937 qp->s_acked = 0;
1938 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
1939 qp->s_draining = 0;
1940 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1941 }
1942 if (priv->s_flags & HFI1_S_TID_WAIT_INTERLCK) {
1943 priv->s_flags &= ~HFI1_S_TID_WAIT_INTERLCK;
1944 hfi1_schedule_send(qp);
1945 }
1946 return wqe;
1947}
1948
1949static void set_restart_qp(struct rvt_qp *qp, struct hfi1_ctxtdata *rcd)
1950{
1951
1952 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
1953 qp->r_flags |= RVT_R_RDMAR_SEQ;
1954 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0);
1955 if (list_empty(&qp->rspwait)) {
1956 qp->r_flags |= RVT_R_RSP_SEND;
1957 rvt_get_qp(qp);
1958 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1959 }
1960 }
1961}
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974static void update_qp_retry_state(struct rvt_qp *qp, u32 psn, u32 spsn,
1975 u32 lpsn)
1976{
1977 struct hfi1_qp_priv *qpriv = qp->priv;
1978
1979 qp->s_psn = psn + 1;
1980
1981
1982
1983
1984
1985
1986 if (cmp_psn(psn, lpsn) >= 0) {
1987 qp->s_cur = qpriv->s_tid_cur + 1;
1988 if (qp->s_cur >= qp->s_size)
1989 qp->s_cur = 0;
1990 qp->s_state = TID_OP(WRITE_REQ);
1991 } else if (!cmp_psn(psn, spsn)) {
1992 qp->s_cur = qpriv->s_tid_cur;
1993 qp->s_state = TID_OP(WRITE_RESP);
1994 }
1995}
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
2009 u64 val, struct hfi1_ctxtdata *rcd)
2010{
2011 struct hfi1_ibport *ibp;
2012 enum ib_wc_status status;
2013 struct hfi1_qp_priv *qpriv = qp->priv;
2014 struct rvt_swqe *wqe;
2015 int ret = 0;
2016 u32 ack_psn;
2017 int diff;
2018 struct rvt_dev_info *rdi;
2019
2020 lockdep_assert_held(&qp->s_lock);
2021
2022
2023
2024
2025
2026
2027 ack_psn = psn;
2028 if (aeth >> IB_AETH_NAK_SHIFT)
2029 ack_psn--;
2030 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2031 ibp = rcd_to_iport(rcd);
2032
2033
2034
2035
2036
2037 while ((diff = delta_psn(ack_psn, wqe->lpsn)) >= 0) {
2038
2039
2040
2041
2042
2043
2044 if (wqe->wr.opcode == IB_WR_RDMA_READ &&
2045 opcode == OP(RDMA_READ_RESPONSE_ONLY) &&
2046 diff == 0) {
2047 ret = 1;
2048 goto bail_stop;
2049 }
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059 if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
2060 (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) ||
2061 (wqe->wr.opcode == IB_WR_TID_RDMA_READ &&
2062 (opcode != TID_OP(READ_RESP) || diff != 0)) ||
2063 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2064 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
2065 (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0)) ||
2066 (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE &&
2067 (delta_psn(psn, qp->s_last_psn) != 1))) {
2068 set_restart_qp(qp, rcd);
2069
2070
2071
2072
2073 goto bail_stop;
2074 }
2075 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2076 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2077 u64 *vaddr = wqe->sg_list[0].vaddr;
2078 *vaddr = val;
2079 }
2080 if (wqe->wr.opcode == IB_WR_OPFN)
2081 opfn_conn_reply(qp, val);
2082
2083 if (qp->s_num_rd_atomic &&
2084 (wqe->wr.opcode == IB_WR_RDMA_READ ||
2085 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2086 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
2087 qp->s_num_rd_atomic--;
2088
2089 if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
2090 !qp->s_num_rd_atomic) {
2091 qp->s_flags &= ~(RVT_S_WAIT_FENCE |
2092 RVT_S_WAIT_ACK);
2093 hfi1_schedule_send(qp);
2094 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) {
2095 qp->s_flags &= ~(RVT_S_WAIT_RDMAR |
2096 RVT_S_WAIT_ACK);
2097 hfi1_schedule_send(qp);
2098 }
2099 }
2100
2101
2102
2103
2104
2105 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE)
2106 break;
2107
2108 wqe = do_rc_completion(qp, wqe, ibp);
2109 if (qp->s_acked == qp->s_tail)
2110 break;
2111 }
2112
2113 trace_hfi1_rc_ack_do(qp, aeth, psn, wqe);
2114 trace_hfi1_sender_do_rc_ack(qp);
2115 switch (aeth >> IB_AETH_NAK_SHIFT) {
2116 case 0:
2117 this_cpu_inc(*ibp->rvp.rc_acks);
2118 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ) {
2119 if (wqe_to_tid_req(wqe)->ack_pending)
2120 rvt_mod_retry_timer_ext(qp,
2121 qpriv->timeout_shift);
2122 else
2123 rvt_stop_rc_timers(qp);
2124 } else if (qp->s_acked != qp->s_tail) {
2125 struct rvt_swqe *__w = NULL;
2126
2127 if (qpriv->s_tid_cur != HFI1_QP_WQE_INVALID)
2128 __w = rvt_get_swqe_ptr(qp, qpriv->s_tid_cur);
2129
2130
2131
2132
2133
2134 if (__w && __w->wr.opcode == IB_WR_TID_RDMA_WRITE &&
2135 opcode == TID_OP(WRITE_RESP)) {
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148 if (cmp_psn(psn, qp->s_last_psn + 1)) {
2149 set_restart_qp(qp, rcd);
2150 goto bail_stop;
2151 }
2152
2153
2154
2155
2156 if (qp->s_cur != qp->s_tail &&
2157 cmp_psn(qp->s_psn, psn) <= 0)
2158 update_qp_retry_state(qp, psn,
2159 __w->psn,
2160 __w->lpsn);
2161 else if (--qpriv->pending_tid_w_resp)
2162 rvt_mod_retry_timer(qp);
2163 else
2164 rvt_stop_rc_timers(qp);
2165 } else {
2166
2167
2168
2169
2170 rvt_mod_retry_timer(qp);
2171
2172
2173
2174
2175
2176 if (cmp_psn(qp->s_psn, psn) <= 0)
2177 reset_psn(qp, psn + 1);
2178 }
2179 } else {
2180
2181 rvt_stop_rc_timers(qp);
2182 if (cmp_psn(qp->s_psn, psn) <= 0) {
2183 qp->s_state = OP(SEND_LAST);
2184 qp->s_psn = psn + 1;
2185 }
2186 }
2187 if (qp->s_flags & RVT_S_WAIT_ACK) {
2188 qp->s_flags &= ~RVT_S_WAIT_ACK;
2189 hfi1_schedule_send(qp);
2190 }
2191 rvt_get_credit(qp, aeth);
2192 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
2193 qp->s_retry = qp->s_retry_cnt;
2194
2195
2196
2197
2198
2199 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE &&
2200 opcode != TID_OP(WRITE_RESP) &&
2201 cmp_psn(psn, wqe->psn) >= 0)
2202 return 1;
2203 update_last_psn(qp, psn);
2204 return 1;
2205
2206 case 1:
2207 ibp->rvp.n_rnr_naks++;
2208 if (qp->s_acked == qp->s_tail)
2209 goto bail_stop;
2210 if (qp->s_flags & RVT_S_WAIT_RNR)
2211 goto bail_stop;
2212 rdi = ib_to_rvt(qp->ibqp.device);
2213 if (qp->s_rnr_retry == 0 &&
2214 !((rdi->post_parms[wqe->wr.opcode].flags &
2215 RVT_OPERATION_IGN_RNR_CNT) &&
2216 qp->s_rnr_retry_cnt == 0)) {
2217 status = IB_WC_RNR_RETRY_EXC_ERR;
2218 goto class_b;
2219 }
2220 if (qp->s_rnr_retry_cnt < 7 && qp->s_rnr_retry_cnt > 0)
2221 qp->s_rnr_retry--;
2222
2223
2224
2225
2226
2227
2228
2229 if (wqe->wr.opcode == IB_WR_TID_RDMA_WRITE) {
2230 reset_psn(qp, qp->s_last_psn + 1);
2231 } else {
2232 update_last_psn(qp, psn - 1);
2233 reset_psn(qp, psn);
2234 }
2235
2236 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn);
2237 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK);
2238 rvt_stop_rc_timers(qp);
2239 rvt_add_rnr_timer(qp, aeth);
2240 return 0;
2241
2242 case 3:
2243 if (qp->s_acked == qp->s_tail)
2244 goto bail_stop;
2245
2246 update_last_psn(qp, psn - 1);
2247 switch ((aeth >> IB_AETH_CREDIT_SHIFT) &
2248 IB_AETH_CREDIT_MASK) {
2249 case 0:
2250 ibp->rvp.n_seq_naks++;
2251
2252
2253
2254
2255
2256
2257 hfi1_restart_rc(qp, psn, 0);
2258 hfi1_schedule_send(qp);
2259 break;
2260
2261 case 1:
2262 status = IB_WC_REM_INV_REQ_ERR;
2263 ibp->rvp.n_other_naks++;
2264 goto class_b;
2265
2266 case 2:
2267 status = IB_WC_REM_ACCESS_ERR;
2268 ibp->rvp.n_other_naks++;
2269 goto class_b;
2270
2271 case 3:
2272 status = IB_WC_REM_OP_ERR;
2273 ibp->rvp.n_other_naks++;
2274class_b:
2275 if (qp->s_last == qp->s_acked) {
2276 if (wqe->wr.opcode == IB_WR_TID_RDMA_READ)
2277 hfi1_kern_read_tid_flow_free(qp);
2278
2279 hfi1_trdma_send_complete(qp, wqe, status);
2280 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
2281 }
2282 break;
2283
2284 default:
2285
2286 goto reserved;
2287 }
2288 qp->s_retry = qp->s_retry_cnt;
2289 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
2290 goto bail_stop;
2291
2292 default:
2293reserved:
2294
2295 goto bail_stop;
2296 }
2297
2298bail_stop:
2299 rvt_stop_rc_timers(qp);
2300 return ret;
2301}
2302
2303
2304
2305
2306
2307static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn,
2308 struct hfi1_ctxtdata *rcd)
2309{
2310 struct rvt_swqe *wqe;
2311
2312 lockdep_assert_held(&qp->s_lock);
2313
2314 rvt_stop_rc_timers(qp);
2315
2316 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2317
2318 while (cmp_psn(psn, wqe->lpsn) > 0) {
2319 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
2320 wqe->wr.opcode == IB_WR_TID_RDMA_READ ||
2321 wqe->wr.opcode == IB_WR_TID_RDMA_WRITE ||
2322 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
2323 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
2324 break;
2325 wqe = do_rc_completion(qp, wqe, ibp);
2326 }
2327
2328 ibp->rvp.n_rdma_seq++;
2329 qp->r_flags |= RVT_R_RDMAR_SEQ;
2330 hfi1_restart_rc(qp, qp->s_last_psn + 1, 0);
2331 if (list_empty(&qp->rspwait)) {
2332 qp->r_flags |= RVT_R_RSP_SEND;
2333 rvt_get_qp(qp);
2334 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2335 }
2336}
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346static void rc_rcv_resp(struct hfi1_packet *packet)
2347{
2348 struct hfi1_ctxtdata *rcd = packet->rcd;
2349 void *data = packet->payload;
2350 u32 tlen = packet->tlen;
2351 struct rvt_qp *qp = packet->qp;
2352 struct hfi1_ibport *ibp;
2353 struct ib_other_headers *ohdr = packet->ohdr;
2354 struct rvt_swqe *wqe;
2355 enum ib_wc_status status;
2356 unsigned long flags;
2357 int diff;
2358 u64 val;
2359 u32 aeth;
2360 u32 psn = ib_bth_get_psn(packet->ohdr);
2361 u32 pmtu = qp->pmtu;
2362 u16 hdrsize = packet->hlen;
2363 u8 opcode = packet->opcode;
2364 u8 pad = packet->pad;
2365 u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2);
2366
2367 spin_lock_irqsave(&qp->s_lock, flags);
2368 trace_hfi1_ack(qp, psn);
2369
2370
2371 if (cmp_psn(psn, READ_ONCE(qp->s_next_psn)) >= 0)
2372 goto ack_done;
2373
2374
2375 diff = cmp_psn(psn, qp->s_last_psn);
2376 if (unlikely(diff <= 0)) {
2377
2378 if (diff == 0 && opcode == OP(ACKNOWLEDGE)) {
2379 aeth = be32_to_cpu(ohdr->u.aeth);
2380 if ((aeth >> IB_AETH_NAK_SHIFT) == 0)
2381 rvt_get_credit(qp, aeth);
2382 }
2383 goto ack_done;
2384 }
2385
2386
2387
2388
2389
2390 if (qp->r_flags & RVT_R_RDMAR_SEQ) {
2391 if (cmp_psn(psn, qp->s_last_psn + 1) != 0)
2392 goto ack_done;
2393 qp->r_flags &= ~RVT_R_RDMAR_SEQ;
2394 }
2395
2396 if (unlikely(qp->s_acked == qp->s_tail))
2397 goto ack_done;
2398 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2399 status = IB_WC_SUCCESS;
2400
2401 switch (opcode) {
2402 case OP(ACKNOWLEDGE):
2403 case OP(ATOMIC_ACKNOWLEDGE):
2404 case OP(RDMA_READ_RESPONSE_FIRST):
2405 aeth = be32_to_cpu(ohdr->u.aeth);
2406 if (opcode == OP(ATOMIC_ACKNOWLEDGE))
2407 val = ib_u64_get(&ohdr->u.at.atomic_ack_eth);
2408 else
2409 val = 0;
2410 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
2411 opcode != OP(RDMA_READ_RESPONSE_FIRST))
2412 goto ack_done;
2413 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2414 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
2415 goto ack_op_err;
2416
2417
2418
2419
2420
2421 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
2422 wqe, psn, pmtu);
2423 goto read_middle;
2424
2425 case OP(RDMA_READ_RESPONSE_MIDDLE):
2426
2427 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1)))
2428 goto ack_seq_err;
2429 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
2430 goto ack_op_err;
2431read_middle:
2432 if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
2433 goto ack_len_err;
2434 if (unlikely(pmtu >= qp->s_rdma_read_len))
2435 goto ack_len_err;
2436
2437
2438
2439
2440
2441 rvt_mod_retry_timer(qp);
2442 if (qp->s_flags & RVT_S_WAIT_ACK) {
2443 qp->s_flags &= ~RVT_S_WAIT_ACK;
2444 hfi1_schedule_send(qp);
2445 }
2446
2447 if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE))
2448 qp->s_retry = qp->s_retry_cnt;
2449
2450
2451
2452
2453
2454 qp->s_rdma_read_len -= pmtu;
2455 update_last_psn(qp, psn);
2456 spin_unlock_irqrestore(&qp->s_lock, flags);
2457 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
2458 data, pmtu, false, false);
2459 goto bail;
2460
2461 case OP(RDMA_READ_RESPONSE_ONLY):
2462 aeth = be32_to_cpu(ohdr->u.aeth);
2463 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
2464 goto ack_done;
2465
2466
2467
2468
2469 if (unlikely(tlen < (hdrsize + extra_bytes)))
2470 goto ack_len_err;
2471
2472
2473
2474
2475
2476 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
2477 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
2478 wqe, psn, pmtu);
2479 goto read_last;
2480
2481 case OP(RDMA_READ_RESPONSE_LAST):
2482
2483 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1)))
2484 goto ack_seq_err;
2485 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
2486 goto ack_op_err;
2487
2488
2489
2490
2491 if (unlikely(tlen <= (hdrsize + extra_bytes)))
2492 goto ack_len_err;
2493read_last:
2494 tlen -= hdrsize + extra_bytes;
2495 if (unlikely(tlen != qp->s_rdma_read_len))
2496 goto ack_len_err;
2497 aeth = be32_to_cpu(ohdr->u.aeth);
2498 rvt_copy_sge(qp, &qp->s_rdma_read_sge,
2499 data, tlen, false, false);
2500 WARN_ON(qp->s_rdma_read_sge.num_sge);
2501 (void)do_rc_ack(qp, aeth, psn,
2502 OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
2503 goto ack_done;
2504 }
2505
2506ack_op_err:
2507 status = IB_WC_LOC_QP_OP_ERR;
2508 goto ack_err;
2509
2510ack_seq_err:
2511 ibp = rcd_to_iport(rcd);
2512 rdma_seq_err(qp, ibp, psn, rcd);
2513 goto ack_done;
2514
2515ack_len_err:
2516 status = IB_WC_LOC_LEN_ERR;
2517ack_err:
2518 if (qp->s_last == qp->s_acked) {
2519 rvt_send_complete(qp, wqe, status);
2520 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
2521 }
2522ack_done:
2523 spin_unlock_irqrestore(&qp->s_lock, flags);
2524bail:
2525 return;
2526}
2527
2528static inline void rc_cancel_ack(struct rvt_qp *qp)
2529{
2530 qp->r_adefered = 0;
2531 if (list_empty(&qp->rspwait))
2532 return;
2533 list_del_init(&qp->rspwait);
2534 qp->r_flags &= ~RVT_R_RSP_NAK;
2535 rvt_put_qp(qp);
2536}
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553static noinline int rc_rcv_error(struct ib_other_headers *ohdr, void *data,
2554 struct rvt_qp *qp, u32 opcode, u32 psn,
2555 int diff, struct hfi1_ctxtdata *rcd)
2556{
2557 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
2558 struct rvt_ack_entry *e;
2559 unsigned long flags;
2560 u8 prev;
2561 u8 mra;
2562 bool old_req;
2563
2564 trace_hfi1_rcv_error(qp, psn);
2565 if (diff > 0) {
2566
2567
2568
2569
2570
2571 if (!qp->r_nak_state) {
2572 ibp->rvp.n_rc_seqnak++;
2573 qp->r_nak_state = IB_NAK_PSN_ERROR;
2574
2575 qp->r_ack_psn = qp->r_psn;
2576
2577
2578
2579
2580
2581 rc_defered_ack(rcd, qp);
2582 }
2583 goto done;
2584 }
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602 e = NULL;
2603 old_req = 1;
2604 ibp->rvp.n_rc_dupreq++;
2605
2606 spin_lock_irqsave(&qp->s_lock, flags);
2607
2608 e = find_prev_entry(qp, psn, &prev, &mra, &old_req);
2609
2610 switch (opcode) {
2611 case OP(RDMA_READ_REQUEST): {
2612 struct ib_reth *reth;
2613 u32 offset;
2614 u32 len;
2615
2616
2617
2618
2619
2620 if (!e || e->opcode != OP(RDMA_READ_REQUEST))
2621 goto unlock_done;
2622
2623 reth = &ohdr->u.rc.reth;
2624
2625
2626
2627
2628
2629
2630
2631 offset = delta_psn(psn, e->psn) * qp->pmtu;
2632 len = be32_to_cpu(reth->length);
2633 if (unlikely(offset + len != e->rdma_sge.sge_length))
2634 goto unlock_done;
2635 release_rdma_sge_mr(e);
2636 if (len != 0) {
2637 u32 rkey = be32_to_cpu(reth->rkey);
2638 u64 vaddr = get_ib_reth_vaddr(reth);
2639 int ok;
2640
2641 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
2642 IB_ACCESS_REMOTE_READ);
2643 if (unlikely(!ok))
2644 goto unlock_done;
2645 } else {
2646 e->rdma_sge.vaddr = NULL;
2647 e->rdma_sge.length = 0;
2648 e->rdma_sge.sge_length = 0;
2649 }
2650 e->psn = psn;
2651 if (old_req)
2652 goto unlock_done;
2653 if (qp->s_acked_ack_queue == qp->s_tail_ack_queue)
2654 qp->s_acked_ack_queue = prev;
2655 qp->s_tail_ack_queue = prev;
2656 break;
2657 }
2658
2659 case OP(COMPARE_SWAP):
2660 case OP(FETCH_ADD): {
2661
2662
2663
2664
2665
2666 if (!e || e->opcode != (u8)opcode || old_req)
2667 goto unlock_done;
2668 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue)
2669 qp->s_acked_ack_queue = prev;
2670 qp->s_tail_ack_queue = prev;
2671 break;
2672 }
2673
2674 default:
2675
2676
2677
2678
2679 if (!(psn & IB_BTH_REQ_ACK) || old_req)
2680 goto unlock_done;
2681
2682
2683
2684
2685 if (mra == qp->r_head_ack_queue) {
2686 spin_unlock_irqrestore(&qp->s_lock, flags);
2687 qp->r_nak_state = 0;
2688 qp->r_ack_psn = qp->r_psn - 1;
2689 goto send_ack;
2690 }
2691
2692
2693
2694
2695
2696 if (qp->s_tail_ack_queue == qp->s_acked_ack_queue)
2697 qp->s_acked_ack_queue = mra;
2698 qp->s_tail_ack_queue = mra;
2699 break;
2700 }
2701 qp->s_ack_state = OP(ACKNOWLEDGE);
2702 qp->s_flags |= RVT_S_RESP_PENDING;
2703 qp->r_nak_state = 0;
2704 hfi1_schedule_send(qp);
2705
2706unlock_done:
2707 spin_unlock_irqrestore(&qp->s_lock, flags);
2708done:
2709 return 1;
2710
2711send_ack:
2712 return 0;
2713}
2714
2715static void log_cca_event(struct hfi1_pportdata *ppd, u8 sl, u32 rlid,
2716 u32 lqpn, u32 rqpn, u8 svc_type)
2717{
2718 struct opa_hfi1_cong_log_event_internal *cc_event;
2719 unsigned long flags;
2720
2721 if (sl >= OPA_MAX_SLS)
2722 return;
2723
2724 spin_lock_irqsave(&ppd->cc_log_lock, flags);
2725
2726 ppd->threshold_cong_event_map[sl / 8] |= 1 << (sl % 8);
2727 ppd->threshold_event_counter++;
2728
2729 cc_event = &ppd->cc_events[ppd->cc_log_idx++];
2730 if (ppd->cc_log_idx == OPA_CONG_LOG_ELEMS)
2731 ppd->cc_log_idx = 0;
2732 cc_event->lqpn = lqpn & RVT_QPN_MASK;
2733 cc_event->rqpn = rqpn & RVT_QPN_MASK;
2734 cc_event->sl = sl;
2735 cc_event->svc_type = svc_type;
2736 cc_event->rlid = rlid;
2737
2738 cc_event->timestamp = ktime_get_ns() / 1024;
2739
2740 spin_unlock_irqrestore(&ppd->cc_log_lock, flags);
2741}
2742
2743void process_becn(struct hfi1_pportdata *ppd, u8 sl, u32 rlid, u32 lqpn,
2744 u32 rqpn, u8 svc_type)
2745{
2746 struct cca_timer *cca_timer;
2747 u16 ccti, ccti_incr, ccti_timer, ccti_limit;
2748 u8 trigger_threshold;
2749 struct cc_state *cc_state;
2750 unsigned long flags;
2751
2752 if (sl >= OPA_MAX_SLS)
2753 return;
2754
2755 cc_state = get_cc_state(ppd);
2756
2757 if (!cc_state)
2758 return;
2759
2760
2761
2762
2763
2764
2765 ccti_limit = cc_state->cct.ccti_limit;
2766 ccti_incr = cc_state->cong_setting.entries[sl].ccti_increase;
2767 ccti_timer = cc_state->cong_setting.entries[sl].ccti_timer;
2768 trigger_threshold =
2769 cc_state->cong_setting.entries[sl].trigger_threshold;
2770
2771 spin_lock_irqsave(&ppd->cca_timer_lock, flags);
2772
2773 cca_timer = &ppd->cca_timer[sl];
2774 if (cca_timer->ccti < ccti_limit) {
2775 if (cca_timer->ccti + ccti_incr <= ccti_limit)
2776 cca_timer->ccti += ccti_incr;
2777 else
2778 cca_timer->ccti = ccti_limit;
2779 set_link_ipg(ppd);
2780 }
2781
2782 ccti = cca_timer->ccti;
2783
2784 if (!hrtimer_active(&cca_timer->hrtimer)) {
2785
2786 unsigned long nsec = 1024 * ccti_timer;
2787
2788 hrtimer_start(&cca_timer->hrtimer, ns_to_ktime(nsec),
2789 HRTIMER_MODE_REL_PINNED);
2790 }
2791
2792 spin_unlock_irqrestore(&ppd->cca_timer_lock, flags);
2793
2794 if ((trigger_threshold != 0) && (ccti >= trigger_threshold))
2795 log_cca_event(ppd, sl, rlid, lqpn, rqpn, svc_type);
2796}
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806void hfi1_rc_rcv(struct hfi1_packet *packet)
2807{
2808 struct hfi1_ctxtdata *rcd = packet->rcd;
2809 void *data = packet->payload;
2810 u32 tlen = packet->tlen;
2811 struct rvt_qp *qp = packet->qp;
2812 struct hfi1_qp_priv *qpriv = qp->priv;
2813 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
2814 struct ib_other_headers *ohdr = packet->ohdr;
2815 u32 opcode = packet->opcode;
2816 u32 hdrsize = packet->hlen;
2817 u32 psn = ib_bth_get_psn(packet->ohdr);
2818 u32 pad = packet->pad;
2819 struct ib_wc wc;
2820 u32 pmtu = qp->pmtu;
2821 int diff;
2822 struct ib_reth *reth;
2823 unsigned long flags;
2824 int ret;
2825 bool copy_last = false, fecn;
2826 u32 rkey;
2827 u8 extra_bytes = pad + packet->extra_byte + (SIZE_OF_CRC << 2);
2828
2829 lockdep_assert_held(&qp->r_lock);
2830
2831 if (hfi1_ruc_check_hdr(ibp, packet))
2832 return;
2833
2834 fecn = process_ecn(qp, packet);
2835 opfn_trigger_conn_request(qp, be32_to_cpu(ohdr->bth[1]));
2836
2837
2838
2839
2840
2841
2842
2843 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
2844 opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
2845 rc_rcv_resp(packet);
2846 return;
2847 }
2848
2849
2850 diff = delta_psn(psn, qp->r_psn);
2851 if (unlikely(diff)) {
2852 if (rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
2853 return;
2854 goto send_ack;
2855 }
2856
2857
2858 switch (qp->r_state) {
2859 case OP(SEND_FIRST):
2860 case OP(SEND_MIDDLE):
2861 if (opcode == OP(SEND_MIDDLE) ||
2862 opcode == OP(SEND_LAST) ||
2863 opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
2864 opcode == OP(SEND_LAST_WITH_INVALIDATE))
2865 break;
2866 goto nack_inv;
2867
2868 case OP(RDMA_WRITE_FIRST):
2869 case OP(RDMA_WRITE_MIDDLE):
2870 if (opcode == OP(RDMA_WRITE_MIDDLE) ||
2871 opcode == OP(RDMA_WRITE_LAST) ||
2872 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
2873 break;
2874 goto nack_inv;
2875
2876 default:
2877 if (opcode == OP(SEND_MIDDLE) ||
2878 opcode == OP(SEND_LAST) ||
2879 opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
2880 opcode == OP(SEND_LAST_WITH_INVALIDATE) ||
2881 opcode == OP(RDMA_WRITE_MIDDLE) ||
2882 opcode == OP(RDMA_WRITE_LAST) ||
2883 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
2884 goto nack_inv;
2885
2886
2887
2888
2889
2890 break;
2891 }
2892
2893 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
2894 rvt_comm_est(qp);
2895
2896
2897 switch (opcode) {
2898 case OP(SEND_FIRST):
2899 ret = rvt_get_rwqe(qp, false);
2900 if (ret < 0)
2901 goto nack_op_err;
2902 if (!ret)
2903 goto rnr_nak;
2904 qp->r_rcv_len = 0;
2905
2906 case OP(SEND_MIDDLE):
2907 case OP(RDMA_WRITE_MIDDLE):
2908send_middle:
2909
2910
2911
2912
2913
2914
2915 if (unlikely(tlen != (hdrsize + pmtu + extra_bytes)))
2916 goto nack_inv;
2917 qp->r_rcv_len += pmtu;
2918 if (unlikely(qp->r_rcv_len > qp->r_len))
2919 goto nack_inv;
2920 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false);
2921 break;
2922
2923 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
2924
2925 ret = rvt_get_rwqe(qp, true);
2926 if (ret < 0)
2927 goto nack_op_err;
2928 if (!ret)
2929 goto rnr_nak;
2930 goto send_last_imm;
2931
2932 case OP(SEND_ONLY):
2933 case OP(SEND_ONLY_WITH_IMMEDIATE):
2934 case OP(SEND_ONLY_WITH_INVALIDATE):
2935 ret = rvt_get_rwqe(qp, false);
2936 if (ret < 0)
2937 goto nack_op_err;
2938 if (!ret)
2939 goto rnr_nak;
2940 qp->r_rcv_len = 0;
2941 if (opcode == OP(SEND_ONLY))
2942 goto no_immediate_data;
2943 if (opcode == OP(SEND_ONLY_WITH_INVALIDATE))
2944 goto send_last_inv;
2945
2946 case OP(SEND_LAST_WITH_IMMEDIATE):
2947send_last_imm:
2948 wc.ex.imm_data = ohdr->u.imm_data;
2949 wc.wc_flags = IB_WC_WITH_IMM;
2950 goto send_last;
2951 case OP(SEND_LAST_WITH_INVALIDATE):
2952send_last_inv:
2953 rkey = be32_to_cpu(ohdr->u.ieth);
2954 if (rvt_invalidate_rkey(qp, rkey))
2955 goto no_immediate_data;
2956 wc.ex.invalidate_rkey = rkey;
2957 wc.wc_flags = IB_WC_WITH_INVALIDATE;
2958 goto send_last;
2959 case OP(RDMA_WRITE_LAST):
2960 copy_last = rvt_is_user_qp(qp);
2961
2962 case OP(SEND_LAST):
2963no_immediate_data:
2964 wc.wc_flags = 0;
2965 wc.ex.imm_data = 0;
2966send_last:
2967
2968
2969 if (unlikely(tlen < (hdrsize + extra_bytes)))
2970 goto nack_inv;
2971
2972 tlen -= (hdrsize + extra_bytes);
2973 wc.byte_len = tlen + qp->r_rcv_len;
2974 if (unlikely(wc.byte_len > qp->r_len))
2975 goto nack_inv;
2976 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, copy_last);
2977 rvt_put_ss(&qp->r_sge);
2978 qp->r_msn++;
2979 if (!__test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
2980 break;
2981 wc.wr_id = qp->r_wr_id;
2982 wc.status = IB_WC_SUCCESS;
2983 if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) ||
2984 opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
2985 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
2986 else
2987 wc.opcode = IB_WC_RECV;
2988 wc.qp = &qp->ibqp;
2989 wc.src_qp = qp->remote_qpn;
2990 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr) & U16_MAX;
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
3003
3004 wc.vendor_err = 0;
3005 wc.pkey_index = 0;
3006 wc.dlid_path_bits = 0;
3007 wc.port_num = 0;
3008
3009 rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr));
3010 break;
3011
3012 case OP(RDMA_WRITE_ONLY):
3013 copy_last = rvt_is_user_qp(qp);
3014
3015 case OP(RDMA_WRITE_FIRST):
3016 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
3017 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
3018 goto nack_inv;
3019
3020 reth = &ohdr->u.rc.reth;
3021 qp->r_len = be32_to_cpu(reth->length);
3022 qp->r_rcv_len = 0;
3023 qp->r_sge.sg_list = NULL;
3024 if (qp->r_len != 0) {
3025 u32 rkey = be32_to_cpu(reth->rkey);
3026 u64 vaddr = get_ib_reth_vaddr(reth);
3027 int ok;
3028
3029
3030 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
3031 rkey, IB_ACCESS_REMOTE_WRITE);
3032 if (unlikely(!ok))
3033 goto nack_acc;
3034 qp->r_sge.num_sge = 1;
3035 } else {
3036 qp->r_sge.num_sge = 0;
3037 qp->r_sge.sge.mr = NULL;
3038 qp->r_sge.sge.vaddr = NULL;
3039 qp->r_sge.sge.length = 0;
3040 qp->r_sge.sge.sge_length = 0;
3041 }
3042 if (opcode == OP(RDMA_WRITE_FIRST))
3043 goto send_middle;
3044 else if (opcode == OP(RDMA_WRITE_ONLY))
3045 goto no_immediate_data;
3046 ret = rvt_get_rwqe(qp, true);
3047 if (ret < 0)
3048 goto nack_op_err;
3049 if (!ret) {
3050
3051 rvt_put_ss(&qp->r_sge);
3052 goto rnr_nak;
3053 }
3054 wc.ex.imm_data = ohdr->u.rc.imm_data;
3055 wc.wc_flags = IB_WC_WITH_IMM;
3056 goto send_last;
3057
3058 case OP(RDMA_READ_REQUEST): {
3059 struct rvt_ack_entry *e;
3060 u32 len;
3061 u8 next;
3062
3063 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
3064 goto nack_inv;
3065 next = qp->r_head_ack_queue + 1;
3066
3067 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
3068 next = 0;
3069 spin_lock_irqsave(&qp->s_lock, flags);
3070 if (unlikely(next == qp->s_acked_ack_queue)) {
3071 if (!qp->s_ack_queue[next].sent)
3072 goto nack_inv_unlck;
3073 update_ack_queue(qp, next);
3074 }
3075 e = &qp->s_ack_queue[qp->r_head_ack_queue];
3076 release_rdma_sge_mr(e);
3077 reth = &ohdr->u.rc.reth;
3078 len = be32_to_cpu(reth->length);
3079 if (len) {
3080 u32 rkey = be32_to_cpu(reth->rkey);
3081 u64 vaddr = get_ib_reth_vaddr(reth);
3082 int ok;
3083
3084
3085 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
3086 rkey, IB_ACCESS_REMOTE_READ);
3087 if (unlikely(!ok))
3088 goto nack_acc_unlck;
3089
3090
3091
3092
3093 qp->r_psn += rvt_div_mtu(qp, len - 1);
3094 } else {
3095 e->rdma_sge.mr = NULL;
3096 e->rdma_sge.vaddr = NULL;
3097 e->rdma_sge.length = 0;
3098 e->rdma_sge.sge_length = 0;
3099 }
3100 e->opcode = opcode;
3101 e->sent = 0;
3102 e->psn = psn;
3103 e->lpsn = qp->r_psn;
3104
3105
3106
3107
3108
3109 qp->r_msn++;
3110 qp->r_psn++;
3111 qp->r_state = opcode;
3112 qp->r_nak_state = 0;
3113 qp->r_head_ack_queue = next;
3114 qpriv->r_tid_alloc = qp->r_head_ack_queue;
3115
3116
3117 qp->s_flags |= RVT_S_RESP_PENDING;
3118 if (fecn)
3119 qp->s_flags |= RVT_S_ECN;
3120 hfi1_schedule_send(qp);
3121
3122 spin_unlock_irqrestore(&qp->s_lock, flags);
3123 return;
3124 }
3125
3126 case OP(COMPARE_SWAP):
3127 case OP(FETCH_ADD): {
3128 struct ib_atomic_eth *ateth = &ohdr->u.atomic_eth;
3129 u64 vaddr = get_ib_ateth_vaddr(ateth);
3130 bool opfn = opcode == OP(COMPARE_SWAP) &&
3131 vaddr == HFI1_VERBS_E_ATOMIC_VADDR;
3132 struct rvt_ack_entry *e;
3133 atomic64_t *maddr;
3134 u64 sdata;
3135 u32 rkey;
3136 u8 next;
3137
3138 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
3139 !opfn))
3140 goto nack_inv;
3141 next = qp->r_head_ack_queue + 1;
3142 if (next > rvt_size_atomic(ib_to_rvt(qp->ibqp.device)))
3143 next = 0;
3144 spin_lock_irqsave(&qp->s_lock, flags);
3145 if (unlikely(next == qp->s_acked_ack_queue)) {
3146 if (!qp->s_ack_queue[next].sent)
3147 goto nack_inv_unlck;
3148 update_ack_queue(qp, next);
3149 }
3150 e = &qp->s_ack_queue[qp->r_head_ack_queue];
3151 release_rdma_sge_mr(e);
3152
3153 if (opfn) {
3154 opfn_conn_response(qp, e, ateth);
3155 goto ack;
3156 }
3157 if (unlikely(vaddr & (sizeof(u64) - 1)))
3158 goto nack_inv_unlck;
3159 rkey = be32_to_cpu(ateth->rkey);
3160
3161 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
3162 vaddr, rkey,
3163 IB_ACCESS_REMOTE_ATOMIC)))
3164 goto nack_acc_unlck;
3165
3166 maddr = (atomic64_t *)qp->r_sge.sge.vaddr;
3167 sdata = get_ib_ateth_swap(ateth);
3168 e->atomic_data = (opcode == OP(FETCH_ADD)) ?
3169 (u64)atomic64_add_return(sdata, maddr) - sdata :
3170 (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr,
3171 get_ib_ateth_compare(ateth),
3172 sdata);
3173 rvt_put_mr(qp->r_sge.sge.mr);
3174 qp->r_sge.num_sge = 0;
3175ack:
3176 e->opcode = opcode;
3177 e->sent = 0;
3178 e->psn = psn;
3179 e->lpsn = psn;
3180 qp->r_msn++;
3181 qp->r_psn++;
3182 qp->r_state = opcode;
3183 qp->r_nak_state = 0;
3184 qp->r_head_ack_queue = next;
3185 qpriv->r_tid_alloc = qp->r_head_ack_queue;
3186
3187
3188 qp->s_flags |= RVT_S_RESP_PENDING;
3189 if (fecn)
3190 qp->s_flags |= RVT_S_ECN;
3191 hfi1_schedule_send(qp);
3192
3193 spin_unlock_irqrestore(&qp->s_lock, flags);
3194 return;
3195 }
3196
3197 default:
3198
3199 goto nack_inv;
3200 }
3201 qp->r_psn++;
3202 qp->r_state = opcode;
3203 qp->r_ack_psn = psn;
3204 qp->r_nak_state = 0;
3205
3206 if (psn & IB_BTH_REQ_ACK || fecn) {
3207 if (packet->numpkt == 0 || fecn ||
3208 qp->r_adefered >= HFI1_PSN_CREDIT) {
3209 rc_cancel_ack(qp);
3210 goto send_ack;
3211 }
3212 qp->r_adefered++;
3213 rc_defered_ack(rcd, qp);
3214 }
3215 return;
3216
3217rnr_nak:
3218 qp->r_nak_state = qp->r_min_rnr_timer | IB_RNR_NAK;
3219 qp->r_ack_psn = qp->r_psn;
3220
3221 rc_defered_ack(rcd, qp);
3222 return;
3223
3224nack_op_err:
3225 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
3226 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
3227 qp->r_ack_psn = qp->r_psn;
3228
3229 rc_defered_ack(rcd, qp);
3230 return;
3231
3232nack_inv_unlck:
3233 spin_unlock_irqrestore(&qp->s_lock, flags);
3234nack_inv:
3235 rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
3236 qp->r_nak_state = IB_NAK_INVALID_REQUEST;
3237 qp->r_ack_psn = qp->r_psn;
3238
3239 rc_defered_ack(rcd, qp);
3240 return;
3241
3242nack_acc_unlck:
3243 spin_unlock_irqrestore(&qp->s_lock, flags);
3244nack_acc:
3245 rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
3246 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
3247 qp->r_ack_psn = qp->r_psn;
3248send_ack:
3249 hfi1_send_rc_ack(packet, fecn);
3250}
3251
3252void hfi1_rc_hdrerr(
3253 struct hfi1_ctxtdata *rcd,
3254 struct hfi1_packet *packet,
3255 struct rvt_qp *qp)
3256{
3257 struct hfi1_ibport *ibp = rcd_to_iport(rcd);
3258 int diff;
3259 u32 opcode;
3260 u32 psn;
3261
3262 if (hfi1_ruc_check_hdr(ibp, packet))
3263 return;
3264
3265 psn = ib_bth_get_psn(packet->ohdr);
3266 opcode = ib_bth_get_opcode(packet->ohdr);
3267
3268
3269 if (opcode < IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) {
3270 diff = delta_psn(psn, qp->r_psn);
3271 if (!qp->r_nak_state && diff >= 0) {
3272 ibp->rvp.n_rc_seqnak++;
3273 qp->r_nak_state = IB_NAK_PSN_ERROR;
3274
3275 qp->r_ack_psn = qp->r_psn;
3276
3277
3278
3279
3280
3281
3282
3283
3284 rc_defered_ack(rcd, qp);
3285 }
3286 }
3287}
3288