1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48#include <linux/io.h>
49#include <rdma/rdma_vt.h>
50#include <rdma/rdmavt_qp.h>
51
52#include "hfi.h"
53#include "qp.h"
54#include "verbs_txreq.h"
55#include "trace.h"
56
57
58#define OP(x) IB_OPCODE_RC_##x
59
60
61
62
63
64
65
66static inline void hfi1_add_retry_timer(struct rvt_qp *qp)
67{
68 struct ib_qp *ibqp = &qp->ibqp;
69 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
70
71 qp->s_flags |= RVT_S_TIMER;
72
73 qp->s_timer.expires = jiffies + qp->timeout_jiffies +
74 rdi->busy_jiffies;
75 add_timer(&qp->s_timer);
76}
77
78
79
80
81
82
83
84
85void hfi1_add_rnr_timer(struct rvt_qp *qp, u32 to)
86{
87 struct hfi1_qp_priv *priv = qp->priv;
88
89 qp->s_flags |= RVT_S_WAIT_RNR;
90 qp->s_timer.expires = jiffies + usecs_to_jiffies(to);
91 add_timer(&priv->s_rnr_timer);
92}
93
94
95
96
97
98
99
100
101static inline void hfi1_mod_retry_timer(struct rvt_qp *qp)
102{
103 struct ib_qp *ibqp = &qp->ibqp;
104 struct rvt_dev_info *rdi = ib_to_rvt(ibqp->device);
105
106 qp->s_flags |= RVT_S_TIMER;
107
108 mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies +
109 rdi->busy_jiffies);
110}
111
112
113
114
115
116
117
118
119static inline int hfi1_stop_retry_timer(struct rvt_qp *qp)
120{
121 int rval = 0;
122
123
124 if (qp->s_flags & RVT_S_TIMER) {
125 qp->s_flags &= ~RVT_S_TIMER;
126 rval = del_timer(&qp->s_timer);
127 }
128 return rval;
129}
130
131
132
133
134
135
136
137void hfi1_stop_rc_timers(struct rvt_qp *qp)
138{
139 struct hfi1_qp_priv *priv = qp->priv;
140
141
142 if (qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR)) {
143 qp->s_flags &= ~(RVT_S_TIMER | RVT_S_WAIT_RNR);
144 del_timer(&qp->s_timer);
145 del_timer(&priv->s_rnr_timer);
146 }
147}
148
149
150
151
152
153
154
155
156static inline int hfi1_stop_rnr_timer(struct rvt_qp *qp)
157{
158 int rval = 0;
159 struct hfi1_qp_priv *priv = qp->priv;
160
161
162 if (qp->s_flags & RVT_S_WAIT_RNR) {
163 qp->s_flags &= ~RVT_S_WAIT_RNR;
164 rval = del_timer(&priv->s_rnr_timer);
165 }
166 return rval;
167}
168
169
170
171
172
173void hfi1_del_timers_sync(struct rvt_qp *qp)
174{
175 struct hfi1_qp_priv *priv = qp->priv;
176
177 del_timer_sync(&qp->s_timer);
178 del_timer_sync(&priv->s_rnr_timer);
179}
180
181
182const u32 rc_only_opcode =
183 BIT(OP(SEND_ONLY) & 0x1f) |
184 BIT(OP(SEND_ONLY_WITH_IMMEDIATE & 0x1f)) |
185 BIT(OP(RDMA_WRITE_ONLY & 0x1f)) |
186 BIT(OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE & 0x1f)) |
187 BIT(OP(RDMA_READ_REQUEST & 0x1f)) |
188 BIT(OP(ACKNOWLEDGE & 0x1f)) |
189 BIT(OP(ATOMIC_ACKNOWLEDGE & 0x1f)) |
190 BIT(OP(COMPARE_SWAP & 0x1f)) |
191 BIT(OP(FETCH_ADD & 0x1f));
192
193static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
194 u32 psn, u32 pmtu)
195{
196 u32 len;
197
198 len = delta_psn(psn, wqe->psn) * pmtu;
199 ss->sge = wqe->sg_list[0];
200 ss->sg_list = wqe->sg_list + 1;
201 ss->num_sge = wqe->wr.num_sge;
202 ss->total_len = wqe->length;
203 hfi1_skip_sge(ss, len, 0);
204 return wqe->length - len;
205}
206
207
208
209
210
211
212
213
214
215
216
217
218static int make_rc_ack(struct hfi1_ibdev *dev, struct rvt_qp *qp,
219 struct hfi1_other_headers *ohdr,
220 struct hfi1_pkt_state *ps)
221{
222 struct rvt_ack_entry *e;
223 u32 hwords;
224 u32 len;
225 u32 bth0;
226 u32 bth2;
227 int middle = 0;
228 u32 pmtu = qp->pmtu;
229 struct hfi1_qp_priv *priv = qp->priv;
230
231
232 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
233 goto bail;
234
235
236 hwords = 5;
237
238 switch (qp->s_ack_state) {
239 case OP(RDMA_READ_RESPONSE_LAST):
240 case OP(RDMA_READ_RESPONSE_ONLY):
241 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
242 if (e->rdma_sge.mr) {
243 rvt_put_mr(e->rdma_sge.mr);
244 e->rdma_sge.mr = NULL;
245 }
246
247 case OP(ATOMIC_ACKNOWLEDGE):
248
249
250
251
252
253 if (++qp->s_tail_ack_queue > HFI1_MAX_RDMA_ATOMIC)
254 qp->s_tail_ack_queue = 0;
255
256 case OP(SEND_ONLY):
257 case OP(ACKNOWLEDGE):
258
259 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
260 if (qp->s_flags & RVT_S_ACK_PENDING)
261 goto normal;
262 goto bail;
263 }
264
265 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
266 if (e->opcode == OP(RDMA_READ_REQUEST)) {
267
268
269
270
271
272
273 len = e->rdma_sge.sge_length;
274 if (len && !e->rdma_sge.mr) {
275 qp->s_tail_ack_queue = qp->r_head_ack_queue;
276 goto bail;
277 }
278
279 ps->s_txreq->mr = e->rdma_sge.mr;
280 if (ps->s_txreq->mr)
281 rvt_get_mr(ps->s_txreq->mr);
282 qp->s_ack_rdma_sge.sge = e->rdma_sge;
283 qp->s_ack_rdma_sge.num_sge = 1;
284 qp->s_cur_sge = &qp->s_ack_rdma_sge;
285 if (len > pmtu) {
286 len = pmtu;
287 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
288 } else {
289 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
290 e->sent = 1;
291 }
292 ohdr->u.aeth = hfi1_compute_aeth(qp);
293 hwords++;
294 qp->s_ack_rdma_psn = e->psn;
295 bth2 = mask_psn(qp->s_ack_rdma_psn++);
296 } else {
297
298 qp->s_cur_sge = NULL;
299 len = 0;
300 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
301 ohdr->u.at.aeth = hfi1_compute_aeth(qp);
302 ohdr->u.at.atomic_ack_eth[0] =
303 cpu_to_be32(e->atomic_data >> 32);
304 ohdr->u.at.atomic_ack_eth[1] =
305 cpu_to_be32(e->atomic_data);
306 hwords += sizeof(ohdr->u.at) / sizeof(u32);
307 bth2 = mask_psn(e->psn);
308 e->sent = 1;
309 }
310 bth0 = qp->s_ack_state << 24;
311 break;
312
313 case OP(RDMA_READ_RESPONSE_FIRST):
314 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
315
316 case OP(RDMA_READ_RESPONSE_MIDDLE):
317 qp->s_cur_sge = &qp->s_ack_rdma_sge;
318 ps->s_txreq->mr = qp->s_ack_rdma_sge.sge.mr;
319 if (ps->s_txreq->mr)
320 rvt_get_mr(ps->s_txreq->mr);
321 len = qp->s_ack_rdma_sge.sge.sge_length;
322 if (len > pmtu) {
323 len = pmtu;
324 middle = HFI1_CAP_IS_KSET(SDMA_AHG);
325 } else {
326 ohdr->u.aeth = hfi1_compute_aeth(qp);
327 hwords++;
328 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
329 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
330 e->sent = 1;
331 }
332 bth0 = qp->s_ack_state << 24;
333 bth2 = mask_psn(qp->s_ack_rdma_psn++);
334 break;
335
336 default:
337normal:
338
339
340
341
342
343
344 qp->s_ack_state = OP(SEND_ONLY);
345 qp->s_flags &= ~RVT_S_ACK_PENDING;
346 qp->s_cur_sge = NULL;
347 if (qp->s_nak_state)
348 ohdr->u.aeth =
349 cpu_to_be32((qp->r_msn & HFI1_MSN_MASK) |
350 (qp->s_nak_state <<
351 HFI1_AETH_CREDIT_SHIFT));
352 else
353 ohdr->u.aeth = hfi1_compute_aeth(qp);
354 hwords++;
355 len = 0;
356 bth0 = OP(ACKNOWLEDGE) << 24;
357 bth2 = mask_psn(qp->s_ack_psn);
358 }
359 qp->s_rdma_ack_cnt++;
360 qp->s_hdrwords = hwords;
361 ps->s_txreq->sde = priv->s_sde;
362 qp->s_cur_size = len;
363 hfi1_make_ruc_header(qp, ohdr, bth0, bth2, middle, ps);
364
365 ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
366 return 1;
367
368bail:
369 qp->s_ack_state = OP(ACKNOWLEDGE);
370
371
372
373
374 smp_wmb();
375 qp->s_flags &= ~(RVT_S_RESP_PENDING
376 | RVT_S_ACK_PENDING
377 | RVT_S_AHG_VALID);
378 return 0;
379}
380
381
382
383
384
385
386
387
388
389int hfi1_make_rc_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps)
390{
391 struct hfi1_qp_priv *priv = qp->priv;
392 struct hfi1_ibdev *dev = to_idev(qp->ibqp.device);
393 struct hfi1_other_headers *ohdr;
394 struct rvt_sge_state *ss;
395 struct rvt_swqe *wqe;
396
397 u32 hwords = 5;
398 u32 len;
399 u32 bth0 = 0;
400 u32 bth2;
401 u32 pmtu = qp->pmtu;
402 char newreq;
403 int middle = 0;
404 int delta;
405
406 ps->s_txreq = get_txreq(ps->dev, qp);
407 if (IS_ERR(ps->s_txreq))
408 goto bail_no_tx;
409
410 ohdr = &ps->s_txreq->phdr.hdr.u.oth;
411 if (qp->remote_ah_attr.ah_flags & IB_AH_GRH)
412 ohdr = &ps->s_txreq->phdr.hdr.u.l.oth;
413
414
415 if ((qp->s_flags & RVT_S_RESP_PENDING) &&
416 make_rc_ack(dev, qp, ohdr, ps))
417 return 1;
418
419 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
420 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
421 goto bail;
422
423 smp_read_barrier_depends();
424 if (qp->s_last == ACCESS_ONCE(qp->s_head))
425 goto bail;
426
427 if (iowait_sdma_pending(&priv->s_iowait)) {
428 qp->s_flags |= RVT_S_WAIT_DMA;
429 goto bail;
430 }
431 clear_ahg(qp);
432 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
433 hfi1_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
434 IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR);
435
436 goto done_free_tx;
437 }
438
439 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK))
440 goto bail;
441
442 if (cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) {
443 if (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
444 qp->s_flags |= RVT_S_WAIT_PSN;
445 goto bail;
446 }
447 qp->s_sending_psn = qp->s_psn;
448 qp->s_sending_hpsn = qp->s_psn - 1;
449 }
450
451
452 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
453 switch (qp->s_state) {
454 default:
455 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK))
456 goto bail;
457
458
459
460
461
462
463
464 newreq = 0;
465 if (qp->s_cur == qp->s_tail) {
466
467 if (qp->s_tail == qp->s_head) {
468 clear_ahg(qp);
469 goto bail;
470 }
471
472
473
474
475 if ((wqe->wr.send_flags & IB_SEND_FENCE) &&
476 qp->s_num_rd_atomic) {
477 qp->s_flags |= RVT_S_WAIT_FENCE;
478 goto bail;
479 }
480 newreq = 1;
481 qp->s_psn = wqe->psn;
482 }
483
484
485
486
487
488 len = wqe->length;
489 ss = &qp->s_sge;
490 bth2 = mask_psn(qp->s_psn);
491 switch (wqe->wr.opcode) {
492 case IB_WR_SEND:
493 case IB_WR_SEND_WITH_IMM:
494
495 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
496 cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
497 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
498 goto bail;
499 }
500 if (len > pmtu) {
501 qp->s_state = OP(SEND_FIRST);
502 len = pmtu;
503 break;
504 }
505 if (wqe->wr.opcode == IB_WR_SEND) {
506 qp->s_state = OP(SEND_ONLY);
507 } else {
508 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
509
510 ohdr->u.imm_data = wqe->wr.ex.imm_data;
511 hwords += 1;
512 }
513 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
514 bth0 |= IB_BTH_SOLICITED;
515 bth2 |= IB_BTH_REQ_ACK;
516 if (++qp->s_cur == qp->s_size)
517 qp->s_cur = 0;
518 break;
519
520 case IB_WR_RDMA_WRITE:
521 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
522 qp->s_lsn++;
523
524 case IB_WR_RDMA_WRITE_WITH_IMM:
525
526 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
527 cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
528 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
529 goto bail;
530 }
531 ohdr->u.rc.reth.vaddr =
532 cpu_to_be64(wqe->rdma_wr.remote_addr);
533 ohdr->u.rc.reth.rkey =
534 cpu_to_be32(wqe->rdma_wr.rkey);
535 ohdr->u.rc.reth.length = cpu_to_be32(len);
536 hwords += sizeof(struct ib_reth) / sizeof(u32);
537 if (len > pmtu) {
538 qp->s_state = OP(RDMA_WRITE_FIRST);
539 len = pmtu;
540 break;
541 }
542 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
543 qp->s_state = OP(RDMA_WRITE_ONLY);
544 } else {
545 qp->s_state =
546 OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
547
548 ohdr->u.rc.imm_data = wqe->wr.ex.imm_data;
549 hwords += 1;
550 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
551 bth0 |= IB_BTH_SOLICITED;
552 }
553 bth2 |= IB_BTH_REQ_ACK;
554 if (++qp->s_cur == qp->s_size)
555 qp->s_cur = 0;
556 break;
557
558 case IB_WR_RDMA_READ:
559
560
561
562
563 if (newreq) {
564 if (qp->s_num_rd_atomic >=
565 qp->s_max_rd_atomic) {
566 qp->s_flags |= RVT_S_WAIT_RDMAR;
567 goto bail;
568 }
569 qp->s_num_rd_atomic++;
570 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
571 qp->s_lsn++;
572 }
573 ohdr->u.rc.reth.vaddr =
574 cpu_to_be64(wqe->rdma_wr.remote_addr);
575 ohdr->u.rc.reth.rkey =
576 cpu_to_be32(wqe->rdma_wr.rkey);
577 ohdr->u.rc.reth.length = cpu_to_be32(len);
578 qp->s_state = OP(RDMA_READ_REQUEST);
579 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
580 ss = NULL;
581 len = 0;
582 bth2 |= IB_BTH_REQ_ACK;
583 if (++qp->s_cur == qp->s_size)
584 qp->s_cur = 0;
585 break;
586
587 case IB_WR_ATOMIC_CMP_AND_SWP:
588 case IB_WR_ATOMIC_FETCH_AND_ADD:
589
590
591
592
593 if (newreq) {
594 if (qp->s_num_rd_atomic >=
595 qp->s_max_rd_atomic) {
596 qp->s_flags |= RVT_S_WAIT_RDMAR;
597 goto bail;
598 }
599 qp->s_num_rd_atomic++;
600 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
601 qp->s_lsn++;
602 }
603 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
604 qp->s_state = OP(COMPARE_SWAP);
605 ohdr->u.atomic_eth.swap_data = cpu_to_be64(
606 wqe->atomic_wr.swap);
607 ohdr->u.atomic_eth.compare_data = cpu_to_be64(
608 wqe->atomic_wr.compare_add);
609 } else {
610 qp->s_state = OP(FETCH_ADD);
611 ohdr->u.atomic_eth.swap_data = cpu_to_be64(
612 wqe->atomic_wr.compare_add);
613 ohdr->u.atomic_eth.compare_data = 0;
614 }
615 ohdr->u.atomic_eth.vaddr[0] = cpu_to_be32(
616 wqe->atomic_wr.remote_addr >> 32);
617 ohdr->u.atomic_eth.vaddr[1] = cpu_to_be32(
618 wqe->atomic_wr.remote_addr);
619 ohdr->u.atomic_eth.rkey = cpu_to_be32(
620 wqe->atomic_wr.rkey);
621 hwords += sizeof(struct ib_atomic_eth) / sizeof(u32);
622 ss = NULL;
623 len = 0;
624 bth2 |= IB_BTH_REQ_ACK;
625 if (++qp->s_cur == qp->s_size)
626 qp->s_cur = 0;
627 break;
628
629 default:
630 goto bail;
631 }
632 qp->s_sge.sge = wqe->sg_list[0];
633 qp->s_sge.sg_list = wqe->sg_list + 1;
634 qp->s_sge.num_sge = wqe->wr.num_sge;
635 qp->s_sge.total_len = wqe->length;
636 qp->s_len = wqe->length;
637 if (newreq) {
638 qp->s_tail++;
639 if (qp->s_tail >= qp->s_size)
640 qp->s_tail = 0;
641 }
642 if (wqe->wr.opcode == IB_WR_RDMA_READ)
643 qp->s_psn = wqe->lpsn + 1;
644 else
645 qp->s_psn++;
646 break;
647
648 case OP(RDMA_READ_RESPONSE_FIRST):
649
650
651
652
653
654
655
656
657
658 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
659
660 case OP(SEND_FIRST):
661 qp->s_state = OP(SEND_MIDDLE);
662
663 case OP(SEND_MIDDLE):
664 bth2 = mask_psn(qp->s_psn++);
665 ss = &qp->s_sge;
666 len = qp->s_len;
667 if (len > pmtu) {
668 len = pmtu;
669 middle = HFI1_CAP_IS_KSET(SDMA_AHG);
670 break;
671 }
672 if (wqe->wr.opcode == IB_WR_SEND) {
673 qp->s_state = OP(SEND_LAST);
674 } else {
675 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
676
677 ohdr->u.imm_data = wqe->wr.ex.imm_data;
678 hwords += 1;
679 }
680 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
681 bth0 |= IB_BTH_SOLICITED;
682 bth2 |= IB_BTH_REQ_ACK;
683 qp->s_cur++;
684 if (qp->s_cur >= qp->s_size)
685 qp->s_cur = 0;
686 break;
687
688 case OP(RDMA_READ_RESPONSE_LAST):
689
690
691
692
693
694
695
696
697
698 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
699
700 case OP(RDMA_WRITE_FIRST):
701 qp->s_state = OP(RDMA_WRITE_MIDDLE);
702
703 case OP(RDMA_WRITE_MIDDLE):
704 bth2 = mask_psn(qp->s_psn++);
705 ss = &qp->s_sge;
706 len = qp->s_len;
707 if (len > pmtu) {
708 len = pmtu;
709 middle = HFI1_CAP_IS_KSET(SDMA_AHG);
710 break;
711 }
712 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) {
713 qp->s_state = OP(RDMA_WRITE_LAST);
714 } else {
715 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
716
717 ohdr->u.imm_data = wqe->wr.ex.imm_data;
718 hwords += 1;
719 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
720 bth0 |= IB_BTH_SOLICITED;
721 }
722 bth2 |= IB_BTH_REQ_ACK;
723 qp->s_cur++;
724 if (qp->s_cur >= qp->s_size)
725 qp->s_cur = 0;
726 break;
727
728 case OP(RDMA_READ_RESPONSE_MIDDLE):
729
730
731
732
733
734
735
736
737
738 len = (delta_psn(qp->s_psn, wqe->psn)) * pmtu;
739 ohdr->u.rc.reth.vaddr =
740 cpu_to_be64(wqe->rdma_wr.remote_addr + len);
741 ohdr->u.rc.reth.rkey =
742 cpu_to_be32(wqe->rdma_wr.rkey);
743 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
744 qp->s_state = OP(RDMA_READ_REQUEST);
745 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
746 bth2 = mask_psn(qp->s_psn) | IB_BTH_REQ_ACK;
747 qp->s_psn = wqe->lpsn + 1;
748 ss = NULL;
749 len = 0;
750 qp->s_cur++;
751 if (qp->s_cur == qp->s_size)
752 qp->s_cur = 0;
753 break;
754 }
755 qp->s_sending_hpsn = bth2;
756 delta = delta_psn(bth2, wqe->psn);
757 if (delta && delta % HFI1_PSN_CREDIT == 0)
758 bth2 |= IB_BTH_REQ_ACK;
759 if (qp->s_flags & RVT_S_SEND_ONE) {
760 qp->s_flags &= ~RVT_S_SEND_ONE;
761 qp->s_flags |= RVT_S_WAIT_ACK;
762 bth2 |= IB_BTH_REQ_ACK;
763 }
764 qp->s_len -= len;
765 qp->s_hdrwords = hwords;
766 ps->s_txreq->sde = priv->s_sde;
767 qp->s_cur_sge = ss;
768 qp->s_cur_size = len;
769 hfi1_make_ruc_header(
770 qp,
771 ohdr,
772 bth0 | (qp->s_state << 24),
773 bth2,
774 middle,
775 ps);
776
777 ps->s_txreq->hdr_dwords = qp->s_hdrwords + 2;
778 return 1;
779
780done_free_tx:
781 hfi1_put_txreq(ps->s_txreq);
782 ps->s_txreq = NULL;
783 return 1;
784
785bail:
786 hfi1_put_txreq(ps->s_txreq);
787
788bail_no_tx:
789 ps->s_txreq = NULL;
790 qp->s_flags &= ~RVT_S_BUSY;
791 qp->s_hdrwords = 0;
792 return 0;
793}
794
795
796
797
798
799
800
801
802
803void hfi1_send_rc_ack(struct hfi1_ctxtdata *rcd, struct rvt_qp *qp,
804 int is_fecn)
805{
806 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
807 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
808 u64 pbc, pbc_flags = 0;
809 u16 lrh0;
810 u16 sc5;
811 u32 bth0;
812 u32 hwords;
813 u32 vl, plen;
814 struct send_context *sc;
815 struct pio_buf *pbuf;
816 struct hfi1_ib_header hdr;
817 struct hfi1_other_headers *ohdr;
818 unsigned long flags;
819
820
821 if (qp->s_flags & RVT_S_RESP_PENDING)
822 goto queue_ack;
823
824
825 smp_read_barrier_depends();
826 if (qp->s_rdma_ack_cnt)
827 goto queue_ack;
828
829
830
831 hwords = 6;
832 if (unlikely(qp->remote_ah_attr.ah_flags & IB_AH_GRH)) {
833 hwords += hfi1_make_grh(ibp, &hdr.u.l.grh,
834 &qp->remote_ah_attr.grh, hwords, 0);
835 ohdr = &hdr.u.l.oth;
836 lrh0 = HFI1_LRH_GRH;
837 } else {
838 ohdr = &hdr.u.oth;
839 lrh0 = HFI1_LRH_BTH;
840 }
841
842 bth0 = hfi1_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24);
843 if (qp->s_mig_state == IB_MIG_MIGRATED)
844 bth0 |= IB_BTH_MIG_REQ;
845 if (qp->r_nak_state)
846 ohdr->u.aeth = cpu_to_be32((qp->r_msn & HFI1_MSN_MASK) |
847 (qp->r_nak_state <<
848 HFI1_AETH_CREDIT_SHIFT));
849 else
850 ohdr->u.aeth = hfi1_compute_aeth(qp);
851 sc5 = ibp->sl_to_sc[qp->remote_ah_attr.sl];
852
853 pbc_flags |= ((!!(sc5 & 0x10)) << PBC_DC_INFO_SHIFT);
854 lrh0 |= (sc5 & 0xf) << 12 | (qp->remote_ah_attr.sl & 0xf) << 4;
855 hdr.lrh[0] = cpu_to_be16(lrh0);
856 hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
857 hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
858 hdr.lrh[3] = cpu_to_be16(ppd->lid | qp->remote_ah_attr.src_path_bits);
859 ohdr->bth[0] = cpu_to_be32(bth0);
860 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
861 ohdr->bth[1] |= cpu_to_be32((!!is_fecn) << HFI1_BECN_SHIFT);
862 ohdr->bth[2] = cpu_to_be32(mask_psn(qp->r_ack_psn));
863
864
865 if (driver_lstate(ppd) != IB_PORT_ACTIVE)
866 return;
867
868 sc = rcd->sc;
869 plen = 2 + hwords;
870 vl = sc_to_vlt(ppd->dd, sc5);
871 pbc = create_pbc(ppd, pbc_flags, qp->srate_mbps, vl, plen);
872
873 pbuf = sc_buffer_alloc(sc, plen, NULL, NULL);
874 if (!pbuf) {
875
876
877
878
879
880
881 goto queue_ack;
882 }
883
884 trace_ack_output_ibhdr(dd_from_ibdev(qp->ibqp.device), &hdr);
885
886
887 ppd->dd->pio_inline_send(ppd->dd, pbuf, pbc, &hdr, hwords);
888
889 return;
890
891queue_ack:
892 this_cpu_inc(*ibp->rvp.rc_qacks);
893 spin_lock_irqsave(&qp->s_lock, flags);
894 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
895 qp->s_nak_state = qp->r_nak_state;
896 qp->s_ack_psn = qp->r_ack_psn;
897 if (is_fecn)
898 qp->s_flags |= RVT_S_ECN;
899
900
901 hfi1_schedule_send(qp);
902 spin_unlock_irqrestore(&qp->s_lock, flags);
903}
904
905
906
907
908
909
910
911
912
913
914static void reset_psn(struct rvt_qp *qp, u32 psn)
915{
916 u32 n = qp->s_acked;
917 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
918 u32 opcode;
919
920 qp->s_cur = n;
921
922
923
924
925
926 if (cmp_psn(psn, wqe->psn) <= 0) {
927 qp->s_state = OP(SEND_LAST);
928 goto done;
929 }
930
931
932 opcode = wqe->wr.opcode;
933 for (;;) {
934 int diff;
935
936 if (++n == qp->s_size)
937 n = 0;
938 if (n == qp->s_tail)
939 break;
940 wqe = rvt_get_swqe_ptr(qp, n);
941 diff = cmp_psn(psn, wqe->psn);
942 if (diff < 0)
943 break;
944 qp->s_cur = n;
945
946
947
948
949 if (diff == 0) {
950 qp->s_state = OP(SEND_LAST);
951 goto done;
952 }
953 opcode = wqe->wr.opcode;
954 }
955
956
957
958
959
960
961 switch (opcode) {
962 case IB_WR_SEND:
963 case IB_WR_SEND_WITH_IMM:
964 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
965 break;
966
967 case IB_WR_RDMA_WRITE:
968 case IB_WR_RDMA_WRITE_WITH_IMM:
969 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
970 break;
971
972 case IB_WR_RDMA_READ:
973 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
974 break;
975
976 default:
977
978
979
980
981 qp->s_state = OP(SEND_LAST);
982 }
983done:
984 qp->s_psn = psn;
985
986
987
988
989
990 if ((cmp_psn(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
991 (cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
992 qp->s_flags |= RVT_S_WAIT_PSN;
993 qp->s_flags &= ~RVT_S_AHG_VALID;
994}
995
996
997
998
999
1000static void restart_rc(struct rvt_qp *qp, u32 psn, int wait)
1001{
1002 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1003 struct hfi1_ibport *ibp;
1004
1005 if (qp->s_retry == 0) {
1006 if (qp->s_mig_state == IB_MIG_ARMED) {
1007 hfi1_migrate_qp(qp);
1008 qp->s_retry = qp->s_retry_cnt;
1009 } else if (qp->s_last == qp->s_acked) {
1010 hfi1_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
1011 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1012 return;
1013 } else {
1014 return;
1015 }
1016 } else {
1017 qp->s_retry--;
1018 }
1019
1020 ibp = to_iport(qp->ibqp.device, qp->port_num);
1021 if (wqe->wr.opcode == IB_WR_RDMA_READ)
1022 ibp->rvp.n_rc_resends++;
1023 else
1024 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn);
1025
1026 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR |
1027 RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_PSN |
1028 RVT_S_WAIT_ACK);
1029 if (wait)
1030 qp->s_flags |= RVT_S_SEND_ONE;
1031 reset_psn(qp, psn);
1032}
1033
1034
1035
1036
1037void hfi1_rc_timeout(unsigned long arg)
1038{
1039 struct rvt_qp *qp = (struct rvt_qp *)arg;
1040 struct hfi1_ibport *ibp;
1041 unsigned long flags;
1042
1043 spin_lock_irqsave(&qp->r_lock, flags);
1044 spin_lock(&qp->s_lock);
1045 if (qp->s_flags & RVT_S_TIMER) {
1046 ibp = to_iport(qp->ibqp.device, qp->port_num);
1047 ibp->rvp.n_rc_timeouts++;
1048 qp->s_flags &= ~RVT_S_TIMER;
1049 del_timer(&qp->s_timer);
1050 trace_hfi1_rc_timeout(qp, qp->s_last_psn + 1);
1051 restart_rc(qp, qp->s_last_psn + 1, 1);
1052 hfi1_schedule_send(qp);
1053 }
1054 spin_unlock(&qp->s_lock);
1055 spin_unlock_irqrestore(&qp->r_lock, flags);
1056}
1057
1058
1059
1060
1061void hfi1_rc_rnr_retry(unsigned long arg)
1062{
1063 struct rvt_qp *qp = (struct rvt_qp *)arg;
1064 unsigned long flags;
1065
1066 spin_lock_irqsave(&qp->s_lock, flags);
1067 hfi1_stop_rnr_timer(qp);
1068 hfi1_schedule_send(qp);
1069 spin_unlock_irqrestore(&qp->s_lock, flags);
1070}
1071
1072
1073
1074
1075
1076static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
1077{
1078 struct rvt_swqe *wqe;
1079 u32 n = qp->s_last;
1080
1081
1082 for (;;) {
1083 wqe = rvt_get_swqe_ptr(qp, n);
1084 if (cmp_psn(psn, wqe->lpsn) <= 0) {
1085 if (wqe->wr.opcode == IB_WR_RDMA_READ)
1086 qp->s_sending_psn = wqe->lpsn + 1;
1087 else
1088 qp->s_sending_psn = psn + 1;
1089 break;
1090 }
1091 if (++n == qp->s_size)
1092 n = 0;
1093 if (n == qp->s_tail)
1094 break;
1095 }
1096}
1097
1098
1099
1100
1101void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_ib_header *hdr)
1102{
1103 struct hfi1_other_headers *ohdr;
1104 struct rvt_swqe *wqe;
1105 struct ib_wc wc;
1106 unsigned i;
1107 u32 opcode;
1108 u32 psn;
1109
1110 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_OR_FLUSH_SEND))
1111 return;
1112
1113
1114 if ((be16_to_cpu(hdr->lrh[0]) & 3) == HFI1_LRH_BTH)
1115 ohdr = &hdr->u.oth;
1116 else
1117 ohdr = &hdr->u.l.oth;
1118
1119 opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
1120 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
1121 opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
1122 WARN_ON(!qp->s_rdma_ack_cnt);
1123 qp->s_rdma_ack_cnt--;
1124 return;
1125 }
1126
1127 psn = be32_to_cpu(ohdr->bth[2]);
1128 reset_sending_psn(qp, psn);
1129
1130
1131
1132
1133
1134 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail &&
1135 !(qp->s_flags &
1136 (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) &&
1137 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
1138 hfi1_add_retry_timer(qp);
1139
1140 while (qp->s_last != qp->s_acked) {
1141 u32 s_last;
1142
1143 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
1144 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 &&
1145 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
1146 break;
1147 s_last = qp->s_last;
1148 if (++s_last >= qp->s_size)
1149 s_last = 0;
1150 qp->s_last = s_last;
1151
1152 barrier();
1153 for (i = 0; i < wqe->wr.num_sge; i++) {
1154 struct rvt_sge *sge = &wqe->sg_list[i];
1155
1156 rvt_put_mr(sge->mr);
1157 }
1158
1159 if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
1160 (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
1161 memset(&wc, 0, sizeof(wc));
1162 wc.wr_id = wqe->wr.wr_id;
1163 wc.status = IB_WC_SUCCESS;
1164 wc.opcode = ib_hfi1_wc_opcode[wqe->wr.opcode];
1165 wc.byte_len = wqe->length;
1166 wc.qp = &qp->ibqp;
1167 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc, 0);
1168 }
1169 }
1170
1171
1172
1173
1174 trace_hfi1_rc_sendcomplete(qp, psn);
1175 if (qp->s_flags & RVT_S_WAIT_PSN &&
1176 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1177 qp->s_flags &= ~RVT_S_WAIT_PSN;
1178 qp->s_sending_psn = qp->s_psn;
1179 qp->s_sending_hpsn = qp->s_psn - 1;
1180 hfi1_schedule_send(qp);
1181 }
1182}
1183
1184static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
1185{
1186 qp->s_last_psn = psn;
1187}
1188
1189
1190
1191
1192
1193
1194static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
1195 struct rvt_swqe *wqe,
1196 struct hfi1_ibport *ibp)
1197{
1198 struct ib_wc wc;
1199 unsigned i;
1200
1201
1202
1203
1204
1205
1206 if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 ||
1207 cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
1208 u32 s_last;
1209
1210 for (i = 0; i < wqe->wr.num_sge; i++) {
1211 struct rvt_sge *sge = &wqe->sg_list[i];
1212
1213 rvt_put_mr(sge->mr);
1214 }
1215 s_last = qp->s_last;
1216 if (++s_last >= qp->s_size)
1217 s_last = 0;
1218 qp->s_last = s_last;
1219
1220 barrier();
1221
1222 if (!(qp->s_flags & RVT_S_SIGNAL_REQ_WR) ||
1223 (wqe->wr.send_flags & IB_SEND_SIGNALED)) {
1224 memset(&wc, 0, sizeof(wc));
1225 wc.wr_id = wqe->wr.wr_id;
1226 wc.status = IB_WC_SUCCESS;
1227 wc.opcode = ib_hfi1_wc_opcode[wqe->wr.opcode];
1228 wc.byte_len = wqe->length;
1229 wc.qp = &qp->ibqp;
1230 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.send_cq), &wc, 0);
1231 }
1232 } else {
1233 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
1234
1235 this_cpu_inc(*ibp->rvp.rc_delayed_comp);
1236
1237
1238
1239
1240 if (ppd->dd->flags & HFI1_HAS_SEND_DMA) {
1241 struct sdma_engine *engine;
1242 u8 sc5;
1243
1244
1245 sc5 = ibp->sl_to_sc[qp->remote_ah_attr.sl];
1246 engine = qp_to_sdma_engine(qp, sc5);
1247 sdma_engine_progress_schedule(engine);
1248 }
1249 }
1250
1251 qp->s_retry = qp->s_retry_cnt;
1252 update_last_psn(qp, wqe->lpsn);
1253
1254
1255
1256
1257
1258
1259 if (qp->s_acked == qp->s_cur) {
1260 if (++qp->s_cur >= qp->s_size)
1261 qp->s_cur = 0;
1262 qp->s_acked = qp->s_cur;
1263 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1264 if (qp->s_acked != qp->s_tail) {
1265 qp->s_state = OP(SEND_LAST);
1266 qp->s_psn = wqe->psn;
1267 }
1268 } else {
1269 if (++qp->s_acked >= qp->s_size)
1270 qp->s_acked = 0;
1271 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
1272 qp->s_draining = 0;
1273 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1274 }
1275 return wqe;
1276}
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
1290 u64 val, struct hfi1_ctxtdata *rcd)
1291{
1292 struct hfi1_ibport *ibp;
1293 enum ib_wc_status status;
1294 struct rvt_swqe *wqe;
1295 int ret = 0;
1296 u32 ack_psn;
1297 int diff;
1298 unsigned long to;
1299
1300
1301
1302
1303
1304
1305
1306 ack_psn = psn;
1307 if (aeth >> 29)
1308 ack_psn--;
1309 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1310 ibp = to_iport(qp->ibqp.device, qp->port_num);
1311
1312
1313
1314
1315
1316 while ((diff = delta_psn(ack_psn, wqe->lpsn)) >= 0) {
1317
1318
1319
1320
1321
1322
1323 if (wqe->wr.opcode == IB_WR_RDMA_READ &&
1324 opcode == OP(RDMA_READ_RESPONSE_ONLY) &&
1325 diff == 0) {
1326 ret = 1;
1327 goto bail_stop;
1328 }
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338 if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
1339 (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) ||
1340 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1341 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
1342 (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) {
1343
1344 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
1345 qp->r_flags |= RVT_R_RDMAR_SEQ;
1346 restart_rc(qp, qp->s_last_psn + 1, 0);
1347 if (list_empty(&qp->rspwait)) {
1348 qp->r_flags |= RVT_R_RSP_SEND;
1349 atomic_inc(&qp->refcount);
1350 list_add_tail(&qp->rspwait,
1351 &rcd->qp_wait_list);
1352 }
1353 }
1354
1355
1356
1357
1358 goto bail_stop;
1359 }
1360 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1361 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
1362 u64 *vaddr = wqe->sg_list[0].vaddr;
1363 *vaddr = val;
1364 }
1365 if (qp->s_num_rd_atomic &&
1366 (wqe->wr.opcode == IB_WR_RDMA_READ ||
1367 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1368 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
1369 qp->s_num_rd_atomic--;
1370
1371 if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
1372 !qp->s_num_rd_atomic) {
1373 qp->s_flags &= ~(RVT_S_WAIT_FENCE |
1374 RVT_S_WAIT_ACK);
1375 hfi1_schedule_send(qp);
1376 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) {
1377 qp->s_flags &= ~(RVT_S_WAIT_RDMAR |
1378 RVT_S_WAIT_ACK);
1379 hfi1_schedule_send(qp);
1380 }
1381 }
1382 wqe = do_rc_completion(qp, wqe, ibp);
1383 if (qp->s_acked == qp->s_tail)
1384 break;
1385 }
1386
1387 switch (aeth >> 29) {
1388 case 0:
1389 this_cpu_inc(*ibp->rvp.rc_acks);
1390 if (qp->s_acked != qp->s_tail) {
1391
1392
1393
1394
1395 hfi1_mod_retry_timer(qp);
1396
1397
1398
1399
1400 if (cmp_psn(qp->s_psn, psn) <= 0)
1401 reset_psn(qp, psn + 1);
1402 } else {
1403
1404 hfi1_stop_rc_timers(qp);
1405 if (cmp_psn(qp->s_psn, psn) <= 0) {
1406 qp->s_state = OP(SEND_LAST);
1407 qp->s_psn = psn + 1;
1408 }
1409 }
1410 if (qp->s_flags & RVT_S_WAIT_ACK) {
1411 qp->s_flags &= ~RVT_S_WAIT_ACK;
1412 hfi1_schedule_send(qp);
1413 }
1414 hfi1_get_credit(qp, aeth);
1415 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1416 qp->s_retry = qp->s_retry_cnt;
1417 update_last_psn(qp, psn);
1418 return 1;
1419
1420 case 1:
1421 ibp->rvp.n_rnr_naks++;
1422 if (qp->s_acked == qp->s_tail)
1423 goto bail_stop;
1424 if (qp->s_flags & RVT_S_WAIT_RNR)
1425 goto bail_stop;
1426 if (qp->s_rnr_retry == 0) {
1427 status = IB_WC_RNR_RETRY_EXC_ERR;
1428 goto class_b;
1429 }
1430 if (qp->s_rnr_retry_cnt < 7)
1431 qp->s_rnr_retry--;
1432
1433
1434 update_last_psn(qp, psn - 1);
1435
1436 ibp->rvp.n_rc_resends += delta_psn(qp->s_psn, psn);
1437
1438 reset_psn(qp, psn);
1439
1440 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK);
1441 hfi1_stop_rc_timers(qp);
1442 to =
1443 ib_hfi1_rnr_table[(aeth >> HFI1_AETH_CREDIT_SHIFT) &
1444 HFI1_AETH_CREDIT_MASK];
1445 hfi1_add_rnr_timer(qp, to);
1446 return 0;
1447
1448 case 3:
1449 if (qp->s_acked == qp->s_tail)
1450 goto bail_stop;
1451
1452 update_last_psn(qp, psn - 1);
1453 switch ((aeth >> HFI1_AETH_CREDIT_SHIFT) &
1454 HFI1_AETH_CREDIT_MASK) {
1455 case 0:
1456 ibp->rvp.n_seq_naks++;
1457
1458
1459
1460
1461
1462
1463 restart_rc(qp, psn, 0);
1464 hfi1_schedule_send(qp);
1465 break;
1466
1467 case 1:
1468 status = IB_WC_REM_INV_REQ_ERR;
1469 ibp->rvp.n_other_naks++;
1470 goto class_b;
1471
1472 case 2:
1473 status = IB_WC_REM_ACCESS_ERR;
1474 ibp->rvp.n_other_naks++;
1475 goto class_b;
1476
1477 case 3:
1478 status = IB_WC_REM_OP_ERR;
1479 ibp->rvp.n_other_naks++;
1480class_b:
1481 if (qp->s_last == qp->s_acked) {
1482 hfi1_send_complete(qp, wqe, status);
1483 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1484 }
1485 break;
1486
1487 default:
1488
1489 goto reserved;
1490 }
1491 qp->s_retry = qp->s_retry_cnt;
1492 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1493 goto bail_stop;
1494
1495 default:
1496reserved:
1497
1498 goto bail_stop;
1499 }
1500 return ret;
1501bail_stop:
1502 hfi1_stop_rc_timers(qp);
1503 return ret;
1504}
1505
1506
1507
1508
1509
1510static void rdma_seq_err(struct rvt_qp *qp, struct hfi1_ibport *ibp, u32 psn,
1511 struct hfi1_ctxtdata *rcd)
1512{
1513 struct rvt_swqe *wqe;
1514
1515
1516 hfi1_stop_rc_timers(qp);
1517
1518 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1519
1520 while (cmp_psn(psn, wqe->lpsn) > 0) {
1521 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1522 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1523 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
1524 break;
1525 wqe = do_rc_completion(qp, wqe, ibp);
1526 }
1527
1528 ibp->rvp.n_rdma_seq++;
1529 qp->r_flags |= RVT_R_RDMAR_SEQ;
1530 restart_rc(qp, qp->s_last_psn + 1, 0);
1531 if (list_empty(&qp->rspwait)) {
1532 qp->r_flags |= RVT_R_RSP_SEND;
1533 atomic_inc(&qp->refcount);
1534 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1535 }
1536}
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554static void rc_rcv_resp(struct hfi1_ibport *ibp,
1555 struct hfi1_other_headers *ohdr,
1556 void *data, u32 tlen, struct rvt_qp *qp,
1557 u32 opcode, u32 psn, u32 hdrsize, u32 pmtu,
1558 struct hfi1_ctxtdata *rcd)
1559{
1560 struct rvt_swqe *wqe;
1561 enum ib_wc_status status;
1562 unsigned long flags;
1563 int diff;
1564 u32 pad;
1565 u32 aeth;
1566 u64 val;
1567
1568 spin_lock_irqsave(&qp->s_lock, flags);
1569
1570 trace_hfi1_rc_ack(qp, psn);
1571
1572
1573 smp_read_barrier_depends();
1574 if (cmp_psn(psn, ACCESS_ONCE(qp->s_next_psn)) >= 0)
1575 goto ack_done;
1576
1577
1578 diff = cmp_psn(psn, qp->s_last_psn);
1579 if (unlikely(diff <= 0)) {
1580
1581 if (diff == 0 && opcode == OP(ACKNOWLEDGE)) {
1582 aeth = be32_to_cpu(ohdr->u.aeth);
1583 if ((aeth >> 29) == 0)
1584 hfi1_get_credit(qp, aeth);
1585 }
1586 goto ack_done;
1587 }
1588
1589
1590
1591
1592
1593 if (qp->r_flags & RVT_R_RDMAR_SEQ) {
1594 if (cmp_psn(psn, qp->s_last_psn + 1) != 0)
1595 goto ack_done;
1596 qp->r_flags &= ~RVT_R_RDMAR_SEQ;
1597 }
1598
1599 if (unlikely(qp->s_acked == qp->s_tail))
1600 goto ack_done;
1601 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1602 status = IB_WC_SUCCESS;
1603
1604 switch (opcode) {
1605 case OP(ACKNOWLEDGE):
1606 case OP(ATOMIC_ACKNOWLEDGE):
1607 case OP(RDMA_READ_RESPONSE_FIRST):
1608 aeth = be32_to_cpu(ohdr->u.aeth);
1609 if (opcode == OP(ATOMIC_ACKNOWLEDGE)) {
1610 __be32 *p = ohdr->u.at.atomic_ack_eth;
1611
1612 val = ((u64)be32_to_cpu(p[0]) << 32) |
1613 be32_to_cpu(p[1]);
1614 } else {
1615 val = 0;
1616 }
1617 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
1618 opcode != OP(RDMA_READ_RESPONSE_FIRST))
1619 goto ack_done;
1620 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1621 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1622 goto ack_op_err;
1623
1624
1625
1626
1627
1628 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1629 wqe, psn, pmtu);
1630 goto read_middle;
1631
1632 case OP(RDMA_READ_RESPONSE_MIDDLE):
1633
1634 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1)))
1635 goto ack_seq_err;
1636 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1637 goto ack_op_err;
1638read_middle:
1639 if (unlikely(tlen != (hdrsize + pmtu + 4)))
1640 goto ack_len_err;
1641 if (unlikely(pmtu >= qp->s_rdma_read_len))
1642 goto ack_len_err;
1643
1644
1645
1646
1647
1648 qp->s_flags |= RVT_S_TIMER;
1649 mod_timer(&qp->s_timer, jiffies + qp->timeout_jiffies);
1650 if (qp->s_flags & RVT_S_WAIT_ACK) {
1651 qp->s_flags &= ~RVT_S_WAIT_ACK;
1652 hfi1_schedule_send(qp);
1653 }
1654
1655 if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE))
1656 qp->s_retry = qp->s_retry_cnt;
1657
1658
1659
1660
1661
1662 qp->s_rdma_read_len -= pmtu;
1663 update_last_psn(qp, psn);
1664 spin_unlock_irqrestore(&qp->s_lock, flags);
1665 hfi1_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0, 0);
1666 goto bail;
1667
1668 case OP(RDMA_READ_RESPONSE_ONLY):
1669 aeth = be32_to_cpu(ohdr->u.aeth);
1670 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
1671 goto ack_done;
1672
1673 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1674
1675
1676
1677
1678 if (unlikely(tlen < (hdrsize + pad + 4)))
1679 goto ack_len_err;
1680
1681
1682
1683
1684
1685 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1686 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1687 wqe, psn, pmtu);
1688 goto read_last;
1689
1690 case OP(RDMA_READ_RESPONSE_LAST):
1691
1692 if (unlikely(cmp_psn(psn, qp->s_last_psn + 1)))
1693 goto ack_seq_err;
1694 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1695 goto ack_op_err;
1696
1697 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1698
1699
1700
1701
1702 if (unlikely(tlen <= (hdrsize + pad + 4)))
1703 goto ack_len_err;
1704read_last:
1705 tlen -= hdrsize + pad + 4;
1706 if (unlikely(tlen != qp->s_rdma_read_len))
1707 goto ack_len_err;
1708 aeth = be32_to_cpu(ohdr->u.aeth);
1709 hfi1_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0, 0);
1710 WARN_ON(qp->s_rdma_read_sge.num_sge);
1711 (void)do_rc_ack(qp, aeth, psn,
1712 OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
1713 goto ack_done;
1714 }
1715
1716ack_op_err:
1717 status = IB_WC_LOC_QP_OP_ERR;
1718 goto ack_err;
1719
1720ack_seq_err:
1721 rdma_seq_err(qp, ibp, psn, rcd);
1722 goto ack_done;
1723
1724ack_len_err:
1725 status = IB_WC_LOC_LEN_ERR;
1726ack_err:
1727 if (qp->s_last == qp->s_acked) {
1728 hfi1_send_complete(qp, wqe, status);
1729 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1730 }
1731ack_done:
1732 spin_unlock_irqrestore(&qp->s_lock, flags);
1733bail:
1734 return;
1735}
1736
1737static inline void rc_defered_ack(struct hfi1_ctxtdata *rcd,
1738 struct rvt_qp *qp)
1739{
1740 if (list_empty(&qp->rspwait)) {
1741 qp->r_flags |= RVT_R_RSP_NAK;
1742 atomic_inc(&qp->refcount);
1743 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1744 }
1745}
1746
1747static inline void rc_cancel_ack(struct rvt_qp *qp)
1748{
1749 struct hfi1_qp_priv *priv = qp->priv;
1750
1751 priv->r_adefered = 0;
1752 if (list_empty(&qp->rspwait))
1753 return;
1754 list_del_init(&qp->rspwait);
1755 qp->r_flags &= ~RVT_R_RSP_NAK;
1756 if (atomic_dec_and_test(&qp->refcount))
1757 wake_up(&qp->wait);
1758}
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775static noinline int rc_rcv_error(struct hfi1_other_headers *ohdr, void *data,
1776 struct rvt_qp *qp, u32 opcode, u32 psn,
1777 int diff, struct hfi1_ctxtdata *rcd)
1778{
1779 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
1780 struct rvt_ack_entry *e;
1781 unsigned long flags;
1782 u8 i, prev;
1783 int old_req;
1784
1785 trace_hfi1_rc_rcv_error(qp, psn);
1786 if (diff > 0) {
1787
1788
1789
1790
1791
1792 if (!qp->r_nak_state) {
1793 ibp->rvp.n_rc_seqnak++;
1794 qp->r_nak_state = IB_NAK_PSN_ERROR;
1795
1796 qp->r_ack_psn = qp->r_psn;
1797
1798
1799
1800
1801
1802 rc_defered_ack(rcd, qp);
1803 }
1804 goto done;
1805 }
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823 e = NULL;
1824 old_req = 1;
1825 ibp->rvp.n_rc_dupreq++;
1826
1827 spin_lock_irqsave(&qp->s_lock, flags);
1828
1829 for (i = qp->r_head_ack_queue; ; i = prev) {
1830 if (i == qp->s_tail_ack_queue)
1831 old_req = 0;
1832 if (i)
1833 prev = i - 1;
1834 else
1835 prev = HFI1_MAX_RDMA_ATOMIC;
1836 if (prev == qp->r_head_ack_queue) {
1837 e = NULL;
1838 break;
1839 }
1840 e = &qp->s_ack_queue[prev];
1841 if (!e->opcode) {
1842 e = NULL;
1843 break;
1844 }
1845 if (cmp_psn(psn, e->psn) >= 0) {
1846 if (prev == qp->s_tail_ack_queue &&
1847 cmp_psn(psn, e->lpsn) <= 0)
1848 old_req = 0;
1849 break;
1850 }
1851 }
1852 switch (opcode) {
1853 case OP(RDMA_READ_REQUEST): {
1854 struct ib_reth *reth;
1855 u32 offset;
1856 u32 len;
1857
1858
1859
1860
1861
1862 if (!e || e->opcode != OP(RDMA_READ_REQUEST))
1863 goto unlock_done;
1864
1865 reth = &ohdr->u.rc.reth;
1866
1867
1868
1869
1870
1871
1872
1873 offset = delta_psn(psn, e->psn) * qp->pmtu;
1874 len = be32_to_cpu(reth->length);
1875 if (unlikely(offset + len != e->rdma_sge.sge_length))
1876 goto unlock_done;
1877 if (e->rdma_sge.mr) {
1878 rvt_put_mr(e->rdma_sge.mr);
1879 e->rdma_sge.mr = NULL;
1880 }
1881 if (len != 0) {
1882 u32 rkey = be32_to_cpu(reth->rkey);
1883 u64 vaddr = be64_to_cpu(reth->vaddr);
1884 int ok;
1885
1886 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
1887 IB_ACCESS_REMOTE_READ);
1888 if (unlikely(!ok))
1889 goto unlock_done;
1890 } else {
1891 e->rdma_sge.vaddr = NULL;
1892 e->rdma_sge.length = 0;
1893 e->rdma_sge.sge_length = 0;
1894 }
1895 e->psn = psn;
1896 if (old_req)
1897 goto unlock_done;
1898 qp->s_tail_ack_queue = prev;
1899 break;
1900 }
1901
1902 case OP(COMPARE_SWAP):
1903 case OP(FETCH_ADD): {
1904
1905
1906
1907
1908
1909 if (!e || e->opcode != (u8)opcode || old_req)
1910 goto unlock_done;
1911 qp->s_tail_ack_queue = prev;
1912 break;
1913 }
1914
1915 default:
1916
1917
1918
1919
1920 if (!(psn & IB_BTH_REQ_ACK) || old_req)
1921 goto unlock_done;
1922
1923
1924
1925
1926 if (i == qp->r_head_ack_queue) {
1927 spin_unlock_irqrestore(&qp->s_lock, flags);
1928 qp->r_nak_state = 0;
1929 qp->r_ack_psn = qp->r_psn - 1;
1930 goto send_ack;
1931 }
1932
1933
1934
1935
1936
1937 qp->s_tail_ack_queue = i;
1938 break;
1939 }
1940 qp->s_ack_state = OP(ACKNOWLEDGE);
1941 qp->s_flags |= RVT_S_RESP_PENDING;
1942 qp->r_nak_state = 0;
1943 hfi1_schedule_send(qp);
1944
1945unlock_done:
1946 spin_unlock_irqrestore(&qp->s_lock, flags);
1947done:
1948 return 1;
1949
1950send_ack:
1951 return 0;
1952}
1953
1954void hfi1_rc_error(struct rvt_qp *qp, enum ib_wc_status err)
1955{
1956 unsigned long flags;
1957 int lastwqe;
1958
1959 spin_lock_irqsave(&qp->s_lock, flags);
1960 lastwqe = rvt_error_qp(qp, err);
1961 spin_unlock_irqrestore(&qp->s_lock, flags);
1962
1963 if (lastwqe) {
1964 struct ib_event ev;
1965
1966 ev.device = qp->ibqp.device;
1967 ev.element.qp = &qp->ibqp;
1968 ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
1969 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
1970 }
1971}
1972
1973static inline void update_ack_queue(struct rvt_qp *qp, unsigned n)
1974{
1975 unsigned next;
1976
1977 next = n + 1;
1978 if (next > HFI1_MAX_RDMA_ATOMIC)
1979 next = 0;
1980 qp->s_tail_ack_queue = next;
1981 qp->s_ack_state = OP(ACKNOWLEDGE);
1982}
1983
1984static void log_cca_event(struct hfi1_pportdata *ppd, u8 sl, u32 rlid,
1985 u32 lqpn, u32 rqpn, u8 svc_type)
1986{
1987 struct opa_hfi1_cong_log_event_internal *cc_event;
1988 unsigned long flags;
1989
1990 if (sl >= OPA_MAX_SLS)
1991 return;
1992
1993 spin_lock_irqsave(&ppd->cc_log_lock, flags);
1994
1995 ppd->threshold_cong_event_map[sl / 8] |= 1 << (sl % 8);
1996 ppd->threshold_event_counter++;
1997
1998 cc_event = &ppd->cc_events[ppd->cc_log_idx++];
1999 if (ppd->cc_log_idx == OPA_CONG_LOG_ELEMS)
2000 ppd->cc_log_idx = 0;
2001 cc_event->lqpn = lqpn & RVT_QPN_MASK;
2002 cc_event->rqpn = rqpn & RVT_QPN_MASK;
2003 cc_event->sl = sl;
2004 cc_event->svc_type = svc_type;
2005 cc_event->rlid = rlid;
2006
2007 cc_event->timestamp = ktime_to_ns(ktime_get()) / 1024;
2008
2009 spin_unlock_irqrestore(&ppd->cc_log_lock, flags);
2010}
2011
2012void process_becn(struct hfi1_pportdata *ppd, u8 sl, u16 rlid, u32 lqpn,
2013 u32 rqpn, u8 svc_type)
2014{
2015 struct cca_timer *cca_timer;
2016 u16 ccti, ccti_incr, ccti_timer, ccti_limit;
2017 u8 trigger_threshold;
2018 struct cc_state *cc_state;
2019 unsigned long flags;
2020
2021 if (sl >= OPA_MAX_SLS)
2022 return;
2023
2024 cca_timer = &ppd->cca_timer[sl];
2025
2026 cc_state = get_cc_state(ppd);
2027
2028 if (!cc_state)
2029 return;
2030
2031
2032
2033
2034
2035
2036 ccti_limit = cc_state->cct.ccti_limit;
2037 ccti_incr = cc_state->cong_setting.entries[sl].ccti_increase;
2038 ccti_timer = cc_state->cong_setting.entries[sl].ccti_timer;
2039 trigger_threshold =
2040 cc_state->cong_setting.entries[sl].trigger_threshold;
2041
2042 spin_lock_irqsave(&ppd->cca_timer_lock, flags);
2043
2044 if (cca_timer->ccti < ccti_limit) {
2045 if (cca_timer->ccti + ccti_incr <= ccti_limit)
2046 cca_timer->ccti += ccti_incr;
2047 else
2048 cca_timer->ccti = ccti_limit;
2049 set_link_ipg(ppd);
2050 }
2051
2052 spin_unlock_irqrestore(&ppd->cca_timer_lock, flags);
2053
2054 ccti = cca_timer->ccti;
2055
2056 if (!hrtimer_active(&cca_timer->hrtimer)) {
2057
2058 unsigned long nsec = 1024 * ccti_timer;
2059
2060 hrtimer_start(&cca_timer->hrtimer, ns_to_ktime(nsec),
2061 HRTIMER_MODE_REL);
2062 }
2063
2064 if ((trigger_threshold != 0) && (ccti >= trigger_threshold))
2065 log_cca_event(ppd, sl, rlid, lqpn, rqpn, svc_type);
2066}
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081void hfi1_rc_rcv(struct hfi1_packet *packet)
2082{
2083 struct hfi1_ctxtdata *rcd = packet->rcd;
2084 struct hfi1_ib_header *hdr = packet->hdr;
2085 u32 rcv_flags = packet->rcv_flags;
2086 void *data = packet->ebuf;
2087 u32 tlen = packet->tlen;
2088 struct rvt_qp *qp = packet->qp;
2089 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
2090 struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
2091 struct hfi1_other_headers *ohdr = packet->ohdr;
2092 u32 bth0, opcode;
2093 u32 hdrsize = packet->hlen;
2094 u32 psn;
2095 u32 pad;
2096 struct ib_wc wc;
2097 u32 pmtu = qp->pmtu;
2098 int diff;
2099 struct ib_reth *reth;
2100 unsigned long flags;
2101 u32 bth1;
2102 int ret, is_fecn = 0;
2103 int copy_last = 0;
2104
2105 bth0 = be32_to_cpu(ohdr->bth[0]);
2106 if (hfi1_ruc_check_hdr(ibp, hdr, rcv_flags & HFI1_HAS_GRH, qp, bth0))
2107 return;
2108
2109 bth1 = be32_to_cpu(ohdr->bth[1]);
2110 if (unlikely(bth1 & (HFI1_BECN_SMASK | HFI1_FECN_SMASK))) {
2111 if (bth1 & HFI1_BECN_SMASK) {
2112 u16 rlid = qp->remote_ah_attr.dlid;
2113 u32 lqpn, rqpn;
2114
2115 lqpn = qp->ibqp.qp_num;
2116 rqpn = qp->remote_qpn;
2117 process_becn(
2118 ppd,
2119 qp->remote_ah_attr.sl,
2120 rlid, lqpn, rqpn,
2121 IB_CC_SVCTYPE_RC);
2122 }
2123 is_fecn = bth1 & HFI1_FECN_SMASK;
2124 }
2125
2126 psn = be32_to_cpu(ohdr->bth[2]);
2127 opcode = (bth0 >> 24) & 0xff;
2128
2129
2130
2131
2132
2133
2134
2135 if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
2136 opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
2137 rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn,
2138 hdrsize, pmtu, rcd);
2139 if (is_fecn)
2140 goto send_ack;
2141 return;
2142 }
2143
2144
2145 diff = delta_psn(psn, qp->r_psn);
2146 if (unlikely(diff)) {
2147 if (rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
2148 return;
2149 goto send_ack;
2150 }
2151
2152
2153 switch (qp->r_state) {
2154 case OP(SEND_FIRST):
2155 case OP(SEND_MIDDLE):
2156 if (opcode == OP(SEND_MIDDLE) ||
2157 opcode == OP(SEND_LAST) ||
2158 opcode == OP(SEND_LAST_WITH_IMMEDIATE))
2159 break;
2160 goto nack_inv;
2161
2162 case OP(RDMA_WRITE_FIRST):
2163 case OP(RDMA_WRITE_MIDDLE):
2164 if (opcode == OP(RDMA_WRITE_MIDDLE) ||
2165 opcode == OP(RDMA_WRITE_LAST) ||
2166 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
2167 break;
2168 goto nack_inv;
2169
2170 default:
2171 if (opcode == OP(SEND_MIDDLE) ||
2172 opcode == OP(SEND_LAST) ||
2173 opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
2174 opcode == OP(RDMA_WRITE_MIDDLE) ||
2175 opcode == OP(RDMA_WRITE_LAST) ||
2176 opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
2177 goto nack_inv;
2178
2179
2180
2181
2182
2183 break;
2184 }
2185
2186 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
2187 qp_comm_est(qp);
2188
2189
2190 switch (opcode) {
2191 case OP(SEND_FIRST):
2192 ret = hfi1_rvt_get_rwqe(qp, 0);
2193 if (ret < 0)
2194 goto nack_op_err;
2195 if (!ret)
2196 goto rnr_nak;
2197 qp->r_rcv_len = 0;
2198
2199 case OP(SEND_MIDDLE):
2200 case OP(RDMA_WRITE_MIDDLE):
2201send_middle:
2202
2203 if (unlikely(tlen != (hdrsize + pmtu + 4)))
2204 goto nack_inv;
2205 qp->r_rcv_len += pmtu;
2206 if (unlikely(qp->r_rcv_len > qp->r_len))
2207 goto nack_inv;
2208 hfi1_copy_sge(&qp->r_sge, data, pmtu, 1, 0);
2209 break;
2210
2211 case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
2212
2213 ret = hfi1_rvt_get_rwqe(qp, 1);
2214 if (ret < 0)
2215 goto nack_op_err;
2216 if (!ret)
2217 goto rnr_nak;
2218 goto send_last_imm;
2219
2220 case OP(SEND_ONLY):
2221 case OP(SEND_ONLY_WITH_IMMEDIATE):
2222 ret = hfi1_rvt_get_rwqe(qp, 0);
2223 if (ret < 0)
2224 goto nack_op_err;
2225 if (!ret)
2226 goto rnr_nak;
2227 qp->r_rcv_len = 0;
2228 if (opcode == OP(SEND_ONLY))
2229 goto no_immediate_data;
2230
2231 case OP(SEND_LAST_WITH_IMMEDIATE):
2232send_last_imm:
2233 wc.ex.imm_data = ohdr->u.imm_data;
2234 wc.wc_flags = IB_WC_WITH_IMM;
2235 goto send_last;
2236 case OP(RDMA_WRITE_LAST):
2237 copy_last = ibpd_to_rvtpd(qp->ibqp.pd)->user;
2238
2239 case OP(SEND_LAST):
2240no_immediate_data:
2241 wc.wc_flags = 0;
2242 wc.ex.imm_data = 0;
2243send_last:
2244
2245 pad = (bth0 >> 20) & 3;
2246
2247
2248 if (unlikely(tlen < (hdrsize + pad + 4)))
2249 goto nack_inv;
2250
2251 tlen -= (hdrsize + pad + 4);
2252 wc.byte_len = tlen + qp->r_rcv_len;
2253 if (unlikely(wc.byte_len > qp->r_len))
2254 goto nack_inv;
2255 hfi1_copy_sge(&qp->r_sge, data, tlen, 1, copy_last);
2256 rvt_put_ss(&qp->r_sge);
2257 qp->r_msn++;
2258 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
2259 break;
2260 wc.wr_id = qp->r_wr_id;
2261 wc.status = IB_WC_SUCCESS;
2262 if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) ||
2263 opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
2264 wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
2265 else
2266 wc.opcode = IB_WC_RECV;
2267 wc.qp = &qp->ibqp;
2268 wc.src_qp = qp->remote_qpn;
2269 wc.slid = qp->remote_ah_attr.dlid;
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281 wc.sl = qp->remote_ah_attr.sl;
2282
2283 wc.vendor_err = 0;
2284 wc.pkey_index = 0;
2285 wc.dlid_path_bits = 0;
2286 wc.port_num = 0;
2287
2288 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
2289 (bth0 & IB_BTH_SOLICITED) != 0);
2290 break;
2291
2292 case OP(RDMA_WRITE_ONLY):
2293 copy_last = 1;
2294
2295 case OP(RDMA_WRITE_FIRST):
2296 case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
2297 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
2298 goto nack_inv;
2299
2300 reth = &ohdr->u.rc.reth;
2301 qp->r_len = be32_to_cpu(reth->length);
2302 qp->r_rcv_len = 0;
2303 qp->r_sge.sg_list = NULL;
2304 if (qp->r_len != 0) {
2305 u32 rkey = be32_to_cpu(reth->rkey);
2306 u64 vaddr = be64_to_cpu(reth->vaddr);
2307 int ok;
2308
2309
2310 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
2311 rkey, IB_ACCESS_REMOTE_WRITE);
2312 if (unlikely(!ok))
2313 goto nack_acc;
2314 qp->r_sge.num_sge = 1;
2315 } else {
2316 qp->r_sge.num_sge = 0;
2317 qp->r_sge.sge.mr = NULL;
2318 qp->r_sge.sge.vaddr = NULL;
2319 qp->r_sge.sge.length = 0;
2320 qp->r_sge.sge.sge_length = 0;
2321 }
2322 if (opcode == OP(RDMA_WRITE_FIRST))
2323 goto send_middle;
2324 else if (opcode == OP(RDMA_WRITE_ONLY))
2325 goto no_immediate_data;
2326 ret = hfi1_rvt_get_rwqe(qp, 1);
2327 if (ret < 0)
2328 goto nack_op_err;
2329 if (!ret)
2330 goto rnr_nak;
2331 wc.ex.imm_data = ohdr->u.rc.imm_data;
2332 wc.wc_flags = IB_WC_WITH_IMM;
2333 goto send_last;
2334
2335 case OP(RDMA_READ_REQUEST): {
2336 struct rvt_ack_entry *e;
2337 u32 len;
2338 u8 next;
2339
2340 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
2341 goto nack_inv;
2342 next = qp->r_head_ack_queue + 1;
2343
2344 if (next > HFI1_MAX_RDMA_ATOMIC)
2345 next = 0;
2346 spin_lock_irqsave(&qp->s_lock, flags);
2347 if (unlikely(next == qp->s_tail_ack_queue)) {
2348 if (!qp->s_ack_queue[next].sent)
2349 goto nack_inv_unlck;
2350 update_ack_queue(qp, next);
2351 }
2352 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2353 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
2354 rvt_put_mr(e->rdma_sge.mr);
2355 e->rdma_sge.mr = NULL;
2356 }
2357 reth = &ohdr->u.rc.reth;
2358 len = be32_to_cpu(reth->length);
2359 if (len) {
2360 u32 rkey = be32_to_cpu(reth->rkey);
2361 u64 vaddr = be64_to_cpu(reth->vaddr);
2362 int ok;
2363
2364
2365 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
2366 rkey, IB_ACCESS_REMOTE_READ);
2367 if (unlikely(!ok))
2368 goto nack_acc_unlck;
2369
2370
2371
2372
2373 if (len > pmtu)
2374 qp->r_psn += (len - 1) / pmtu;
2375 } else {
2376 e->rdma_sge.mr = NULL;
2377 e->rdma_sge.vaddr = NULL;
2378 e->rdma_sge.length = 0;
2379 e->rdma_sge.sge_length = 0;
2380 }
2381 e->opcode = opcode;
2382 e->sent = 0;
2383 e->psn = psn;
2384 e->lpsn = qp->r_psn;
2385
2386
2387
2388
2389
2390 qp->r_msn++;
2391 qp->r_psn++;
2392 qp->r_state = opcode;
2393 qp->r_nak_state = 0;
2394 qp->r_head_ack_queue = next;
2395
2396
2397 qp->s_flags |= RVT_S_RESP_PENDING;
2398 hfi1_schedule_send(qp);
2399
2400 spin_unlock_irqrestore(&qp->s_lock, flags);
2401 if (is_fecn)
2402 goto send_ack;
2403 return;
2404 }
2405
2406 case OP(COMPARE_SWAP):
2407 case OP(FETCH_ADD): {
2408 struct ib_atomic_eth *ateth;
2409 struct rvt_ack_entry *e;
2410 u64 vaddr;
2411 atomic64_t *maddr;
2412 u64 sdata;
2413 u32 rkey;
2414 u8 next;
2415
2416 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
2417 goto nack_inv;
2418 next = qp->r_head_ack_queue + 1;
2419 if (next > HFI1_MAX_RDMA_ATOMIC)
2420 next = 0;
2421 spin_lock_irqsave(&qp->s_lock, flags);
2422 if (unlikely(next == qp->s_tail_ack_queue)) {
2423 if (!qp->s_ack_queue[next].sent)
2424 goto nack_inv_unlck;
2425 update_ack_queue(qp, next);
2426 }
2427 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2428 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
2429 rvt_put_mr(e->rdma_sge.mr);
2430 e->rdma_sge.mr = NULL;
2431 }
2432 ateth = &ohdr->u.atomic_eth;
2433 vaddr = ((u64)be32_to_cpu(ateth->vaddr[0]) << 32) |
2434 be32_to_cpu(ateth->vaddr[1]);
2435 if (unlikely(vaddr & (sizeof(u64) - 1)))
2436 goto nack_inv_unlck;
2437 rkey = be32_to_cpu(ateth->rkey);
2438
2439 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
2440 vaddr, rkey,
2441 IB_ACCESS_REMOTE_ATOMIC)))
2442 goto nack_acc_unlck;
2443
2444 maddr = (atomic64_t *)qp->r_sge.sge.vaddr;
2445 sdata = be64_to_cpu(ateth->swap_data);
2446 e->atomic_data = (opcode == OP(FETCH_ADD)) ?
2447 (u64)atomic64_add_return(sdata, maddr) - sdata :
2448 (u64)cmpxchg((u64 *)qp->r_sge.sge.vaddr,
2449 be64_to_cpu(ateth->compare_data),
2450 sdata);
2451 rvt_put_mr(qp->r_sge.sge.mr);
2452 qp->r_sge.num_sge = 0;
2453 e->opcode = opcode;
2454 e->sent = 0;
2455 e->psn = psn;
2456 e->lpsn = psn;
2457 qp->r_msn++;
2458 qp->r_psn++;
2459 qp->r_state = opcode;
2460 qp->r_nak_state = 0;
2461 qp->r_head_ack_queue = next;
2462
2463
2464 qp->s_flags |= RVT_S_RESP_PENDING;
2465 hfi1_schedule_send(qp);
2466
2467 spin_unlock_irqrestore(&qp->s_lock, flags);
2468 if (is_fecn)
2469 goto send_ack;
2470 return;
2471 }
2472
2473 default:
2474
2475 goto nack_inv;
2476 }
2477 qp->r_psn++;
2478 qp->r_state = opcode;
2479 qp->r_ack_psn = psn;
2480 qp->r_nak_state = 0;
2481
2482 if (psn & IB_BTH_REQ_ACK) {
2483 struct hfi1_qp_priv *priv = qp->priv;
2484
2485 if (packet->numpkt == 0) {
2486 rc_cancel_ack(qp);
2487 goto send_ack;
2488 }
2489 if (priv->r_adefered >= HFI1_PSN_CREDIT) {
2490 rc_cancel_ack(qp);
2491 goto send_ack;
2492 }
2493 if (unlikely(is_fecn)) {
2494 rc_cancel_ack(qp);
2495 goto send_ack;
2496 }
2497 priv->r_adefered++;
2498 rc_defered_ack(rcd, qp);
2499 }
2500 return;
2501
2502rnr_nak:
2503 qp->r_nak_state = qp->r_min_rnr_timer | IB_RNR_NAK;
2504 qp->r_ack_psn = qp->r_psn;
2505
2506 rc_defered_ack(rcd, qp);
2507 return;
2508
2509nack_op_err:
2510 hfi1_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2511 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
2512 qp->r_ack_psn = qp->r_psn;
2513
2514 rc_defered_ack(rcd, qp);
2515 return;
2516
2517nack_inv_unlck:
2518 spin_unlock_irqrestore(&qp->s_lock, flags);
2519nack_inv:
2520 hfi1_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2521 qp->r_nak_state = IB_NAK_INVALID_REQUEST;
2522 qp->r_ack_psn = qp->r_psn;
2523
2524 rc_defered_ack(rcd, qp);
2525 return;
2526
2527nack_acc_unlck:
2528 spin_unlock_irqrestore(&qp->s_lock, flags);
2529nack_acc:
2530 hfi1_rc_error(qp, IB_WC_LOC_PROT_ERR);
2531 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
2532 qp->r_ack_psn = qp->r_psn;
2533send_ack:
2534 hfi1_send_rc_ack(rcd, qp, is_fecn);
2535}
2536
2537void hfi1_rc_hdrerr(
2538 struct hfi1_ctxtdata *rcd,
2539 struct hfi1_ib_header *hdr,
2540 u32 rcv_flags,
2541 struct rvt_qp *qp)
2542{
2543 int has_grh = rcv_flags & HFI1_HAS_GRH;
2544 struct hfi1_other_headers *ohdr;
2545 struct hfi1_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
2546 int diff;
2547 u32 opcode;
2548 u32 psn, bth0;
2549
2550
2551 ohdr = &hdr->u.oth;
2552 if (has_grh)
2553 ohdr = &hdr->u.l.oth;
2554
2555 bth0 = be32_to_cpu(ohdr->bth[0]);
2556 if (hfi1_ruc_check_hdr(ibp, hdr, has_grh, qp, bth0))
2557 return;
2558
2559 psn = be32_to_cpu(ohdr->bth[2]);
2560 opcode = (bth0 >> 24) & 0xff;
2561
2562
2563 if (opcode < IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST) {
2564 diff = delta_psn(psn, qp->r_psn);
2565 if (!qp->r_nak_state && diff >= 0) {
2566 ibp->rvp.n_rc_seqnak++;
2567 qp->r_nak_state = IB_NAK_PSN_ERROR;
2568
2569 qp->r_ack_psn = qp->r_psn;
2570
2571
2572
2573
2574
2575
2576
2577
2578 rc_defered_ack(rcd, qp);
2579 }
2580 }
2581}
2582