1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33#include <linux/kernel.h>
34#include <linux/in.h>
35#include <linux/slab.h>
36#include <linux/vmalloc.h>
37#include <linux/ratelimit.h>
38#include <net/addrconf.h>
39
40#include "rds_single_path.h"
41#include "rds.h"
42#include "ib.h"
43
44
45
46
47static void rds_ib_set_protocol(struct rds_connection *conn, unsigned int version)
48{
49 conn->c_version = version;
50}
51
52
53
54
55static void rds_ib_set_flow_control(struct rds_connection *conn, u32 credits)
56{
57 struct rds_ib_connection *ic = conn->c_transport_data;
58
59 if (rds_ib_sysctl_flow_control && credits != 0) {
60
61 ic->i_flowctl = 1;
62 rds_ib_send_add_credits(conn, credits);
63 } else {
64 ic->i_flowctl = 0;
65 }
66}
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82static void
83rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)
84{
85 int ret;
86
87 attr->min_rnr_timer = IB_RNR_TIMER_000_32;
88 ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);
89 if (ret)
90 printk(KERN_NOTICE "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d\n", -ret);
91}
92
93
94
95
96
97void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_event *event)
98{
99 struct rds_ib_connection *ic = conn->c_transport_data;
100 const union rds_ib_conn_priv *dp = NULL;
101 struct ib_qp_attr qp_attr;
102 __be64 ack_seq = 0;
103 __be32 credit = 0;
104 u8 major = 0;
105 u8 minor = 0;
106 int err;
107
108 dp = event->param.conn.private_data;
109 if (conn->c_isv6) {
110 if (event->param.conn.private_data_len >=
111 sizeof(struct rds6_ib_connect_private)) {
112 major = dp->ricp_v6.dp_protocol_major;
113 minor = dp->ricp_v6.dp_protocol_minor;
114 credit = dp->ricp_v6.dp_credit;
115
116
117
118
119
120 ack_seq = get_unaligned(&dp->ricp_v6.dp_ack_seq);
121 }
122 } else if (event->param.conn.private_data_len >=
123 sizeof(struct rds_ib_connect_private)) {
124 major = dp->ricp_v4.dp_protocol_major;
125 minor = dp->ricp_v4.dp_protocol_minor;
126 credit = dp->ricp_v4.dp_credit;
127 ack_seq = get_unaligned(&dp->ricp_v4.dp_ack_seq);
128 }
129
130
131 if (major) {
132 rds_ib_set_protocol(conn, RDS_PROTOCOL(major, minor));
133 rds_ib_set_flow_control(conn, be32_to_cpu(credit));
134 }
135
136 if (conn->c_version < RDS_PROTOCOL_VERSION) {
137 if (conn->c_version != RDS_PROTOCOL_COMPAT_VERSION) {
138 pr_notice("RDS/IB: Connection <%pI6c,%pI6c> version %u.%u no longer supported\n",
139 &conn->c_laddr, &conn->c_faddr,
140 RDS_PROTOCOL_MAJOR(conn->c_version),
141 RDS_PROTOCOL_MINOR(conn->c_version));
142 rds_conn_destroy(conn);
143 return;
144 }
145 }
146
147 pr_notice("RDS/IB: %s conn connected <%pI6c,%pI6c,%d> version %u.%u%s\n",
148 ic->i_active_side ? "Active" : "Passive",
149 &conn->c_laddr, &conn->c_faddr, conn->c_tos,
150 RDS_PROTOCOL_MAJOR(conn->c_version),
151 RDS_PROTOCOL_MINOR(conn->c_version),
152 ic->i_flowctl ? ", flow control" : "");
153
154 atomic_set(&ic->i_cq_quiesce, 0);
155
156
157
158
159
160 rds_ib_send_init_ring(ic);
161 rds_ib_recv_init_ring(ic);
162
163
164 rds_ib_recv_refill(conn, 1, GFP_KERNEL);
165
166
167 rds_ib_tune_rnr(ic, &qp_attr);
168
169 qp_attr.qp_state = IB_QPS_RTS;
170 err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
171 if (err)
172 printk(KERN_NOTICE "ib_modify_qp(IB_QP_STATE, RTS): err=%d\n", err);
173
174
175 err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr);
176 if (err)
177 printk(KERN_ERR "rds_ib_update_ipaddr failed (%d)\n",
178 err);
179
180
181
182 if (dp) {
183 if (ack_seq)
184 rds_send_drop_acked(conn, be64_to_cpu(ack_seq),
185 NULL);
186 }
187
188 conn->c_proposed_version = conn->c_version;
189 rds_connect_complete(conn);
190}
191
192static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
193 struct rdma_conn_param *conn_param,
194 union rds_ib_conn_priv *dp,
195 u32 protocol_version,
196 u32 max_responder_resources,
197 u32 max_initiator_depth,
198 bool isv6)
199{
200 struct rds_ib_connection *ic = conn->c_transport_data;
201 struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
202
203 memset(conn_param, 0, sizeof(struct rdma_conn_param));
204
205 conn_param->responder_resources =
206 min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources);
207 conn_param->initiator_depth =
208 min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth);
209 conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
210 conn_param->rnr_retry_count = 7;
211
212 if (dp) {
213 memset(dp, 0, sizeof(*dp));
214 if (isv6) {
215 dp->ricp_v6.dp_saddr = conn->c_laddr;
216 dp->ricp_v6.dp_daddr = conn->c_faddr;
217 dp->ricp_v6.dp_protocol_major =
218 RDS_PROTOCOL_MAJOR(protocol_version);
219 dp->ricp_v6.dp_protocol_minor =
220 RDS_PROTOCOL_MINOR(protocol_version);
221 dp->ricp_v6.dp_protocol_minor_mask =
222 cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
223 dp->ricp_v6.dp_ack_seq =
224 cpu_to_be64(rds_ib_piggyb_ack(ic));
225 dp->ricp_v6.dp_cmn.ricpc_dp_toss = conn->c_tos;
226
227 conn_param->private_data = &dp->ricp_v6;
228 conn_param->private_data_len = sizeof(dp->ricp_v6);
229 } else {
230 dp->ricp_v4.dp_saddr = conn->c_laddr.s6_addr32[3];
231 dp->ricp_v4.dp_daddr = conn->c_faddr.s6_addr32[3];
232 dp->ricp_v4.dp_protocol_major =
233 RDS_PROTOCOL_MAJOR(protocol_version);
234 dp->ricp_v4.dp_protocol_minor =
235 RDS_PROTOCOL_MINOR(protocol_version);
236 dp->ricp_v4.dp_protocol_minor_mask =
237 cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
238 dp->ricp_v4.dp_ack_seq =
239 cpu_to_be64(rds_ib_piggyb_ack(ic));
240 dp->ricp_v4.dp_cmn.ricpc_dp_toss = conn->c_tos;
241
242 conn_param->private_data = &dp->ricp_v4;
243 conn_param->private_data_len = sizeof(dp->ricp_v4);
244 }
245
246
247 if (ic->i_flowctl) {
248 unsigned int credits;
249
250 credits = IB_GET_POST_CREDITS
251 (atomic_read(&ic->i_credits));
252 if (isv6)
253 dp->ricp_v6.dp_credit = cpu_to_be32(credits);
254 else
255 dp->ricp_v4.dp_credit = cpu_to_be32(credits);
256 atomic_sub(IB_SET_POST_CREDITS(credits),
257 &ic->i_credits);
258 }
259 }
260}
261
262static void rds_ib_cq_event_handler(struct ib_event *event, void *data)
263{
264 rdsdebug("event %u (%s) data %p\n",
265 event->event, ib_event_msg(event->event), data);
266}
267
268
269
270
271
272
273
274
275
276static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
277{
278 struct rds_connection *conn = context;
279 struct rds_ib_connection *ic = conn->c_transport_data;
280
281 rdsdebug("conn %p cq %p\n", conn, cq);
282
283 rds_ib_stats_inc(s_ib_evt_handler_call);
284
285 tasklet_schedule(&ic->i_recv_tasklet);
286}
287
288static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
289 struct ib_wc *wcs)
290{
291 int nr, i;
292 struct ib_wc *wc;
293
294 while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
295 for (i = 0; i < nr; i++) {
296 wc = wcs + i;
297 rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
298 (unsigned long long)wc->wr_id, wc->status,
299 wc->byte_len, be32_to_cpu(wc->ex.imm_data));
300
301 if (wc->wr_id <= ic->i_send_ring.w_nr ||
302 wc->wr_id == RDS_IB_ACK_WR_ID)
303 rds_ib_send_cqe_handler(ic, wc);
304 else
305 rds_ib_mr_cqe_handler(ic, wc);
306
307 }
308 }
309}
310
311static void rds_ib_tasklet_fn_send(unsigned long data)
312{
313 struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
314 struct rds_connection *conn = ic->conn;
315
316 rds_ib_stats_inc(s_ib_tasklet_call);
317
318
319 if (atomic_read(&ic->i_cq_quiesce))
320 return;
321
322 poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
323 ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
324 poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
325
326 if (rds_conn_up(conn) &&
327 (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags) ||
328 test_bit(0, &conn->c_map_queued)))
329 rds_send_xmit(&ic->conn->c_path[0]);
330}
331
332static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
333 struct ib_wc *wcs,
334 struct rds_ib_ack_state *ack_state)
335{
336 int nr, i;
337 struct ib_wc *wc;
338
339 while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
340 for (i = 0; i < nr; i++) {
341 wc = wcs + i;
342 rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
343 (unsigned long long)wc->wr_id, wc->status,
344 wc->byte_len, be32_to_cpu(wc->ex.imm_data));
345
346 rds_ib_recv_cqe_handler(ic, wc, ack_state);
347 }
348 }
349}
350
351static void rds_ib_tasklet_fn_recv(unsigned long data)
352{
353 struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
354 struct rds_connection *conn = ic->conn;
355 struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
356 struct rds_ib_ack_state state;
357
358 if (!rds_ibdev)
359 rds_conn_drop(conn);
360
361 rds_ib_stats_inc(s_ib_tasklet_call);
362
363
364 if (atomic_read(&ic->i_cq_quiesce))
365 return;
366
367 memset(&state, 0, sizeof(state));
368 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
369 ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
370 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
371
372 if (state.ack_next_valid)
373 rds_ib_set_ack(ic, state.ack_next, state.ack_required);
374 if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {
375 rds_send_drop_acked(conn, state.ack_recv, NULL);
376 ic->i_ack_recv = state.ack_recv;
377 }
378
379 if (rds_conn_up(conn))
380 rds_ib_attempt_ack(ic);
381}
382
383static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
384{
385 struct rds_connection *conn = data;
386 struct rds_ib_connection *ic = conn->c_transport_data;
387
388 rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,
389 ib_event_msg(event->event));
390
391 switch (event->event) {
392 case IB_EVENT_COMM_EST:
393 rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
394 break;
395 default:
396 rdsdebug("Fatal QP Event %u (%s) - connection %pI6c->%pI6c, reconnecting\n",
397 event->event, ib_event_msg(event->event),
398 &conn->c_laddr, &conn->c_faddr);
399 rds_conn_drop(conn);
400 break;
401 }
402}
403
404static void rds_ib_cq_comp_handler_send(struct ib_cq *cq, void *context)
405{
406 struct rds_connection *conn = context;
407 struct rds_ib_connection *ic = conn->c_transport_data;
408
409 rdsdebug("conn %p cq %p\n", conn, cq);
410
411 rds_ib_stats_inc(s_ib_evt_handler_call);
412
413 tasklet_schedule(&ic->i_send_tasklet);
414}
415
416static inline int ibdev_get_unused_vector(struct rds_ib_device *rds_ibdev)
417{
418 int min = rds_ibdev->vector_load[rds_ibdev->dev->num_comp_vectors - 1];
419 int index = rds_ibdev->dev->num_comp_vectors - 1;
420 int i;
421
422 for (i = rds_ibdev->dev->num_comp_vectors - 1; i >= 0; i--) {
423 if (rds_ibdev->vector_load[i] < min) {
424 index = i;
425 min = rds_ibdev->vector_load[i];
426 }
427 }
428
429 rds_ibdev->vector_load[index]++;
430 return index;
431}
432
433static inline void ibdev_put_vector(struct rds_ib_device *rds_ibdev, int index)
434{
435 rds_ibdev->vector_load[index]--;
436}
437
438
439
440
441
442static int rds_ib_setup_qp(struct rds_connection *conn)
443{
444 struct rds_ib_connection *ic = conn->c_transport_data;
445 struct ib_device *dev = ic->i_cm_id->device;
446 struct ib_qp_init_attr attr;
447 struct ib_cq_init_attr cq_attr = {};
448 struct rds_ib_device *rds_ibdev;
449 int ret, fr_queue_space;
450
451
452
453
454
455 rds_ibdev = rds_ib_get_client_data(dev);
456 if (!rds_ibdev)
457 return -EOPNOTSUPP;
458
459
460
461
462
463 fr_queue_space = rds_ibdev->use_fastreg ?
464 (RDS_IB_DEFAULT_FR_WR + 1) +
465 (RDS_IB_DEFAULT_FR_INV_WR + 1)
466 : 0;
467
468
469 rds_ib_add_conn(rds_ibdev, conn);
470
471 if (rds_ibdev->max_wrs < ic->i_send_ring.w_nr + 1)
472 rds_ib_ring_resize(&ic->i_send_ring, rds_ibdev->max_wrs - 1);
473 if (rds_ibdev->max_wrs < ic->i_recv_ring.w_nr + 1)
474 rds_ib_ring_resize(&ic->i_recv_ring, rds_ibdev->max_wrs - 1);
475
476
477 ic->i_pd = rds_ibdev->pd;
478
479 ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
480 cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
481 cq_attr.comp_vector = ic->i_scq_vector;
482 ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
483 rds_ib_cq_event_handler, conn,
484 &cq_attr);
485 if (IS_ERR(ic->i_send_cq)) {
486 ret = PTR_ERR(ic->i_send_cq);
487 ic->i_send_cq = NULL;
488 ibdev_put_vector(rds_ibdev, ic->i_scq_vector);
489 rdsdebug("ib_create_cq send failed: %d\n", ret);
490 goto rds_ibdev_out;
491 }
492
493 ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
494 cq_attr.cqe = ic->i_recv_ring.w_nr;
495 cq_attr.comp_vector = ic->i_rcq_vector;
496 ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
497 rds_ib_cq_event_handler, conn,
498 &cq_attr);
499 if (IS_ERR(ic->i_recv_cq)) {
500 ret = PTR_ERR(ic->i_recv_cq);
501 ic->i_recv_cq = NULL;
502 ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);
503 rdsdebug("ib_create_cq recv failed: %d\n", ret);
504 goto send_cq_out;
505 }
506
507 ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
508 if (ret) {
509 rdsdebug("ib_req_notify_cq send failed: %d\n", ret);
510 goto recv_cq_out;
511 }
512
513 ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
514 if (ret) {
515 rdsdebug("ib_req_notify_cq recv failed: %d\n", ret);
516 goto recv_cq_out;
517 }
518
519
520 memset(&attr, 0, sizeof(attr));
521 attr.event_handler = rds_ib_qp_event_handler;
522 attr.qp_context = conn;
523
524 attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1;
525 attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;
526 attr.cap.max_send_sge = rds_ibdev->max_sge;
527 attr.cap.max_recv_sge = RDS_IB_RECV_SGE;
528 attr.sq_sig_type = IB_SIGNAL_REQ_WR;
529 attr.qp_type = IB_QPT_RC;
530 attr.send_cq = ic->i_send_cq;
531 attr.recv_cq = ic->i_recv_cq;
532 atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);
533 atomic_set(&ic->i_fastunreg_wrs, RDS_IB_DEFAULT_FR_INV_WR);
534
535
536
537
538
539 ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
540 if (ret) {
541 rdsdebug("rdma_create_qp failed: %d\n", ret);
542 goto recv_cq_out;
543 }
544
545 ic->i_send_hdrs = ib_dma_alloc_coherent(dev,
546 ic->i_send_ring.w_nr *
547 sizeof(struct rds_header),
548 &ic->i_send_hdrs_dma, GFP_KERNEL);
549 if (!ic->i_send_hdrs) {
550 ret = -ENOMEM;
551 rdsdebug("ib_dma_alloc_coherent send failed\n");
552 goto qp_out;
553 }
554
555 ic->i_recv_hdrs = ib_dma_alloc_coherent(dev,
556 ic->i_recv_ring.w_nr *
557 sizeof(struct rds_header),
558 &ic->i_recv_hdrs_dma, GFP_KERNEL);
559 if (!ic->i_recv_hdrs) {
560 ret = -ENOMEM;
561 rdsdebug("ib_dma_alloc_coherent recv failed\n");
562 goto send_hdrs_dma_out;
563 }
564
565 ic->i_ack = ib_dma_alloc_coherent(dev, sizeof(struct rds_header),
566 &ic->i_ack_dma, GFP_KERNEL);
567 if (!ic->i_ack) {
568 ret = -ENOMEM;
569 rdsdebug("ib_dma_alloc_coherent ack failed\n");
570 goto recv_hdrs_dma_out;
571 }
572
573 ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work),
574 ic->i_send_ring.w_nr),
575 ibdev_to_node(dev));
576 if (!ic->i_sends) {
577 ret = -ENOMEM;
578 rdsdebug("send allocation failed\n");
579 goto ack_dma_out;
580 }
581
582 ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work),
583 ic->i_recv_ring.w_nr),
584 ibdev_to_node(dev));
585 if (!ic->i_recvs) {
586 ret = -ENOMEM;
587 rdsdebug("recv allocation failed\n");
588 goto sends_out;
589 }
590
591 rds_ib_recv_init_ack(ic);
592
593 rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,
594 ic->i_send_cq, ic->i_recv_cq);
595
596 goto out;
597
598sends_out:
599 vfree(ic->i_sends);
600ack_dma_out:
601 ib_dma_free_coherent(dev, sizeof(struct rds_header),
602 ic->i_ack, ic->i_ack_dma);
603recv_hdrs_dma_out:
604 ib_dma_free_coherent(dev, ic->i_recv_ring.w_nr *
605 sizeof(struct rds_header),
606 ic->i_recv_hdrs, ic->i_recv_hdrs_dma);
607send_hdrs_dma_out:
608 ib_dma_free_coherent(dev, ic->i_send_ring.w_nr *
609 sizeof(struct rds_header),
610 ic->i_send_hdrs, ic->i_send_hdrs_dma);
611qp_out:
612 rdma_destroy_qp(ic->i_cm_id);
613recv_cq_out:
614 if (!ib_destroy_cq(ic->i_recv_cq))
615 ic->i_recv_cq = NULL;
616send_cq_out:
617 if (!ib_destroy_cq(ic->i_send_cq))
618 ic->i_send_cq = NULL;
619rds_ibdev_out:
620 rds_ib_remove_conn(rds_ibdev, conn);
621out:
622 rds_ib_dev_put(rds_ibdev);
623
624 return ret;
625}
626
627static u32 rds_ib_protocol_compatible(struct rdma_cm_event *event, bool isv6)
628{
629 const union rds_ib_conn_priv *dp = event->param.conn.private_data;
630 u8 data_len, major, minor;
631 u32 version = 0;
632 __be16 mask;
633 u16 common;
634
635
636
637
638
639
640
641
642
643
644
645
646 if (!event->param.conn.private_data_len) {
647 printk(KERN_NOTICE "RDS incoming connection has no private data, "
648 "rejecting\n");
649 return 0;
650 }
651
652 if (isv6) {
653 data_len = sizeof(struct rds6_ib_connect_private);
654 major = dp->ricp_v6.dp_protocol_major;
655 minor = dp->ricp_v6.dp_protocol_minor;
656 mask = dp->ricp_v6.dp_protocol_minor_mask;
657 } else {
658 data_len = sizeof(struct rds_ib_connect_private);
659 major = dp->ricp_v4.dp_protocol_major;
660 minor = dp->ricp_v4.dp_protocol_minor;
661 mask = dp->ricp_v4.dp_protocol_minor_mask;
662 }
663
664
665 if (event->param.conn.private_data_len < data_len || major == 0)
666 return RDS_PROTOCOL_4_0;
667
668 common = be16_to_cpu(mask) & RDS_IB_SUPPORTED_PROTOCOLS;
669 if (major == 4 && common) {
670 version = RDS_PROTOCOL_4_0;
671 while ((common >>= 1) != 0)
672 version++;
673 } else if (RDS_PROTOCOL_COMPAT_VERSION ==
674 RDS_PROTOCOL(major, minor)) {
675 version = RDS_PROTOCOL_COMPAT_VERSION;
676 } else {
677 if (isv6)
678 printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI6c using incompatible protocol version %u.%u\n",
679 &dp->ricp_v6.dp_saddr, major, minor);
680 else
681 printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI4 using incompatible protocol version %u.%u\n",
682 &dp->ricp_v4.dp_saddr, major, minor);
683 }
684 return version;
685}
686
687#if IS_ENABLED(CONFIG_IPV6)
688
689
690
691
692
693
694
695
696
697
698static u32 __rds_find_ifindex(struct net *net, const struct in6_addr *addr)
699{
700 struct net_device *dev;
701 int idx = 0;
702
703 rcu_read_lock();
704 for_each_netdev_rcu(net, dev) {
705 if (ipv6_chk_addr(net, addr, dev, 1)) {
706 idx = dev->ifindex;
707 break;
708 }
709 }
710 rcu_read_unlock();
711
712 return idx;
713}
714#endif
715
716int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
717 struct rdma_cm_event *event, bool isv6)
718{
719 __be64 lguid = cm_id->route.path_rec->sgid.global.interface_id;
720 __be64 fguid = cm_id->route.path_rec->dgid.global.interface_id;
721 const struct rds_ib_conn_priv_cmn *dp_cmn;
722 struct rds_connection *conn = NULL;
723 struct rds_ib_connection *ic = NULL;
724 struct rdma_conn_param conn_param;
725 const union rds_ib_conn_priv *dp;
726 union rds_ib_conn_priv dp_rep;
727 struct in6_addr s_mapped_addr;
728 struct in6_addr d_mapped_addr;
729 const struct in6_addr *saddr6;
730 const struct in6_addr *daddr6;
731 int destroy = 1;
732 u32 ifindex = 0;
733 u32 version;
734 int err = 1;
735
736
737 version = rds_ib_protocol_compatible(event, isv6);
738 if (!version) {
739 err = RDS_RDMA_REJ_INCOMPAT;
740 goto out;
741 }
742
743 dp = event->param.conn.private_data;
744 if (isv6) {
745#if IS_ENABLED(CONFIG_IPV6)
746 dp_cmn = &dp->ricp_v6.dp_cmn;
747 saddr6 = &dp->ricp_v6.dp_saddr;
748 daddr6 = &dp->ricp_v6.dp_daddr;
749
750
751
752
753 if (ipv6_addr_type(daddr6) & IPV6_ADDR_LINKLOCAL) {
754
755 ifindex = __rds_find_ifindex(&init_net, daddr6);
756
757 if (ifindex == 0) {
758 err = -EOPNOTSUPP;
759 goto out;
760 }
761 } else if (ipv6_addr_type(saddr6) & IPV6_ADDR_LINKLOCAL) {
762
763 ifindex = __rds_find_ifindex(&init_net, daddr6);
764
765 if (ifindex == 0) {
766 err = -EOPNOTSUPP;
767 goto out;
768 }
769 }
770#else
771 err = -EOPNOTSUPP;
772 goto out;
773#endif
774 } else {
775 dp_cmn = &dp->ricp_v4.dp_cmn;
776 ipv6_addr_set_v4mapped(dp->ricp_v4.dp_saddr, &s_mapped_addr);
777 ipv6_addr_set_v4mapped(dp->ricp_v4.dp_daddr, &d_mapped_addr);
778 saddr6 = &s_mapped_addr;
779 daddr6 = &d_mapped_addr;
780 }
781
782 rdsdebug("saddr %pI6c daddr %pI6c RDSv%u.%u lguid 0x%llx fguid 0x%llx, tos:%d\n",
783 saddr6, daddr6, RDS_PROTOCOL_MAJOR(version),
784 RDS_PROTOCOL_MINOR(version),
785 (unsigned long long)be64_to_cpu(lguid),
786 (unsigned long long)be64_to_cpu(fguid), dp_cmn->ricpc_dp_toss);
787
788
789 conn = rds_conn_create(&init_net, daddr6, saddr6,
790 &rds_ib_transport, dp_cmn->ricpc_dp_toss,
791 GFP_KERNEL, ifindex);
792 if (IS_ERR(conn)) {
793 rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn));
794 conn = NULL;
795 goto out;
796 }
797
798
799
800
801
802
803
804
805 mutex_lock(&conn->c_cm_lock);
806 if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
807 if (rds_conn_state(conn) == RDS_CONN_UP) {
808 rdsdebug("incoming connect while connecting\n");
809 rds_conn_drop(conn);
810 rds_ib_stats_inc(s_ib_listen_closed_stale);
811 } else
812 if (rds_conn_state(conn) == RDS_CONN_CONNECTING) {
813
814 rds_ib_stats_inc(s_ib_connect_raced);
815 }
816 goto out;
817 }
818
819 ic = conn->c_transport_data;
820
821 rds_ib_set_protocol(conn, version);
822 rds_ib_set_flow_control(conn, be32_to_cpu(dp_cmn->ricpc_credit));
823
824
825
826 if (dp_cmn->ricpc_ack_seq)
827 rds_send_drop_acked(conn, be64_to_cpu(dp_cmn->ricpc_ack_seq),
828 NULL);
829
830 BUG_ON(cm_id->context);
831 BUG_ON(ic->i_cm_id);
832
833 ic->i_cm_id = cm_id;
834 cm_id->context = conn;
835
836
837
838 destroy = 0;
839
840 err = rds_ib_setup_qp(conn);
841 if (err) {
842 rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", err);
843 goto out;
844 }
845
846 rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
847 event->param.conn.responder_resources,
848 event->param.conn.initiator_depth, isv6);
849
850
851 if (rdma_accept(cm_id, &conn_param))
852 rds_ib_conn_error(conn, "rdma_accept failed\n");
853
854out:
855 if (conn)
856 mutex_unlock(&conn->c_cm_lock);
857 if (err)
858 rdma_reject(cm_id, &err, sizeof(int));
859 return destroy;
860}
861
862
863int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id, bool isv6)
864{
865 struct rds_connection *conn = cm_id->context;
866 struct rds_ib_connection *ic = conn->c_transport_data;
867 struct rdma_conn_param conn_param;
868 union rds_ib_conn_priv dp;
869 int ret;
870
871
872
873 rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1);
874 ic->i_flowctl = rds_ib_sysctl_flow_control;
875
876 ret = rds_ib_setup_qp(conn);
877 if (ret) {
878 rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", ret);
879 goto out;
880 }
881
882 rds_ib_cm_fill_conn_param(conn, &conn_param, &dp,
883 conn->c_proposed_version,
884 UINT_MAX, UINT_MAX, isv6);
885 ret = rdma_connect(cm_id, &conn_param);
886 if (ret)
887 rds_ib_conn_error(conn, "rdma_connect failed (%d)\n", ret);
888
889out:
890
891
892
893 if (ret) {
894 if (ic->i_cm_id == cm_id)
895 ret = 0;
896 }
897 ic->i_active_side = true;
898 return ret;
899}
900
901int rds_ib_conn_path_connect(struct rds_conn_path *cp)
902{
903 struct rds_connection *conn = cp->cp_conn;
904 struct sockaddr_storage src, dest;
905 rdma_cm_event_handler handler;
906 struct rds_ib_connection *ic;
907 int ret;
908
909 ic = conn->c_transport_data;
910
911
912
913#if IS_ENABLED(CONFIG_IPV6)
914 if (conn->c_isv6)
915 handler = rds6_rdma_cm_event_handler;
916 else
917#endif
918 handler = rds_rdma_cm_event_handler;
919 ic->i_cm_id = rdma_create_id(&init_net, handler, conn,
920 RDMA_PS_TCP, IB_QPT_RC);
921 if (IS_ERR(ic->i_cm_id)) {
922 ret = PTR_ERR(ic->i_cm_id);
923 ic->i_cm_id = NULL;
924 rdsdebug("rdma_create_id() failed: %d\n", ret);
925 goto out;
926 }
927
928 rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);
929
930 if (ipv6_addr_v4mapped(&conn->c_faddr)) {
931 struct sockaddr_in *sin;
932
933 sin = (struct sockaddr_in *)&src;
934 sin->sin_family = AF_INET;
935 sin->sin_addr.s_addr = conn->c_laddr.s6_addr32[3];
936 sin->sin_port = 0;
937
938 sin = (struct sockaddr_in *)&dest;
939 sin->sin_family = AF_INET;
940 sin->sin_addr.s_addr = conn->c_faddr.s6_addr32[3];
941 sin->sin_port = htons(RDS_PORT);
942 } else {
943 struct sockaddr_in6 *sin6;
944
945 sin6 = (struct sockaddr_in6 *)&src;
946 sin6->sin6_family = AF_INET6;
947 sin6->sin6_addr = conn->c_laddr;
948 sin6->sin6_port = 0;
949 sin6->sin6_scope_id = conn->c_dev_if;
950
951 sin6 = (struct sockaddr_in6 *)&dest;
952 sin6->sin6_family = AF_INET6;
953 sin6->sin6_addr = conn->c_faddr;
954 sin6->sin6_port = htons(RDS_CM_PORT);
955 sin6->sin6_scope_id = conn->c_dev_if;
956 }
957
958 ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
959 (struct sockaddr *)&dest,
960 RDS_RDMA_RESOLVE_TIMEOUT_MS);
961 if (ret) {
962 rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,
963 ret);
964 rdma_destroy_id(ic->i_cm_id);
965 ic->i_cm_id = NULL;
966 }
967
968out:
969 return ret;
970}
971
972
973
974
975
976
977void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
978{
979 struct rds_connection *conn = cp->cp_conn;
980 struct rds_ib_connection *ic = conn->c_transport_data;
981 int err = 0;
982
983 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,
984 ic->i_pd, ic->i_send_cq, ic->i_recv_cq,
985 ic->i_cm_id ? ic->i_cm_id->qp : NULL);
986
987 if (ic->i_cm_id) {
988 struct ib_device *dev = ic->i_cm_id->device;
989
990 rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
991 err = rdma_disconnect(ic->i_cm_id);
992 if (err) {
993
994
995
996 rdsdebug("failed to disconnect, cm: %p err %d\n",
997 ic->i_cm_id, err);
998 }
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009 wait_event(rds_ib_ring_empty_wait,
1010 rds_ib_ring_empty(&ic->i_recv_ring) &&
1011 (atomic_read(&ic->i_signaled_sends) == 0) &&
1012 (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR) &&
1013 (atomic_read(&ic->i_fastunreg_wrs) == RDS_IB_DEFAULT_FR_INV_WR));
1014 tasklet_kill(&ic->i_send_tasklet);
1015 tasklet_kill(&ic->i_recv_tasklet);
1016
1017 atomic_set(&ic->i_cq_quiesce, 1);
1018
1019
1020 if (ic->i_cm_id->qp)
1021 rdma_destroy_qp(ic->i_cm_id);
1022 if (ic->i_send_cq) {
1023 if (ic->rds_ibdev)
1024 ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector);
1025 ib_destroy_cq(ic->i_send_cq);
1026 }
1027
1028 if (ic->i_recv_cq) {
1029 if (ic->rds_ibdev)
1030 ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector);
1031 ib_destroy_cq(ic->i_recv_cq);
1032 }
1033
1034
1035 if (ic->i_send_hdrs)
1036 ib_dma_free_coherent(dev,
1037 ic->i_send_ring.w_nr *
1038 sizeof(struct rds_header),
1039 ic->i_send_hdrs,
1040 ic->i_send_hdrs_dma);
1041
1042 if (ic->i_recv_hdrs)
1043 ib_dma_free_coherent(dev,
1044 ic->i_recv_ring.w_nr *
1045 sizeof(struct rds_header),
1046 ic->i_recv_hdrs,
1047 ic->i_recv_hdrs_dma);
1048
1049 if (ic->i_ack)
1050 ib_dma_free_coherent(dev, sizeof(struct rds_header),
1051 ic->i_ack, ic->i_ack_dma);
1052
1053 if (ic->i_sends)
1054 rds_ib_send_clear_ring(ic);
1055 if (ic->i_recvs)
1056 rds_ib_recv_clear_ring(ic);
1057
1058 rdma_destroy_id(ic->i_cm_id);
1059
1060
1061
1062
1063 if (ic->rds_ibdev)
1064 rds_ib_remove_conn(ic->rds_ibdev, conn);
1065
1066 ic->i_cm_id = NULL;
1067 ic->i_pd = NULL;
1068 ic->i_send_cq = NULL;
1069 ic->i_recv_cq = NULL;
1070 ic->i_send_hdrs = NULL;
1071 ic->i_recv_hdrs = NULL;
1072 ic->i_ack = NULL;
1073 }
1074 BUG_ON(ic->rds_ibdev);
1075
1076
1077 if (ic->i_data_op) {
1078 struct rds_message *rm;
1079
1080 rm = container_of(ic->i_data_op, struct rds_message, data);
1081 rds_message_put(rm);
1082 ic->i_data_op = NULL;
1083 }
1084
1085
1086 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
1087#ifdef KERNEL_HAS_ATOMIC64
1088 atomic64_set(&ic->i_ack_next, 0);
1089#else
1090 ic->i_ack_next = 0;
1091#endif
1092 ic->i_ack_recv = 0;
1093
1094
1095 ic->i_flowctl = 0;
1096 atomic_set(&ic->i_credits, 0);
1097
1098 rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);
1099 rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);
1100
1101 if (ic->i_ibinc) {
1102 rds_inc_put(&ic->i_ibinc->ii_inc);
1103 ic->i_ibinc = NULL;
1104 }
1105
1106 vfree(ic->i_sends);
1107 ic->i_sends = NULL;
1108 vfree(ic->i_recvs);
1109 ic->i_recvs = NULL;
1110 ic->i_active_side = false;
1111}
1112
1113int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp)
1114{
1115 struct rds_ib_connection *ic;
1116 unsigned long flags;
1117 int ret;
1118
1119
1120 ic = kzalloc(sizeof(struct rds_ib_connection), gfp);
1121 if (!ic)
1122 return -ENOMEM;
1123
1124 ret = rds_ib_recv_alloc_caches(ic, gfp);
1125 if (ret) {
1126 kfree(ic);
1127 return ret;
1128 }
1129
1130 INIT_LIST_HEAD(&ic->ib_node);
1131 tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,
1132 (unsigned long)ic);
1133 tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,
1134 (unsigned long)ic);
1135 mutex_init(&ic->i_recv_mutex);
1136#ifndef KERNEL_HAS_ATOMIC64
1137 spin_lock_init(&ic->i_ack_lock);
1138#endif
1139 atomic_set(&ic->i_signaled_sends, 0);
1140
1141
1142
1143
1144
1145 rds_ib_ring_init(&ic->i_send_ring, rds_ib_sysctl_max_send_wr);
1146 rds_ib_ring_init(&ic->i_recv_ring, rds_ib_sysctl_max_recv_wr);
1147
1148 ic->conn = conn;
1149 conn->c_transport_data = ic;
1150
1151 spin_lock_irqsave(&ib_nodev_conns_lock, flags);
1152 list_add_tail(&ic->ib_node, &ib_nodev_conns);
1153 spin_unlock_irqrestore(&ib_nodev_conns_lock, flags);
1154
1155
1156 rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data);
1157 return 0;
1158}
1159
1160
1161
1162
1163void rds_ib_conn_free(void *arg)
1164{
1165 struct rds_ib_connection *ic = arg;
1166 spinlock_t *lock_ptr;
1167
1168 rdsdebug("ic %p\n", ic);
1169
1170
1171
1172
1173
1174
1175 lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;
1176
1177 spin_lock_irq(lock_ptr);
1178 list_del(&ic->ib_node);
1179 spin_unlock_irq(lock_ptr);
1180
1181 rds_ib_recv_free_caches(ic);
1182
1183 kfree(ic);
1184}
1185
1186
1187
1188
1189
1190void
1191__rds_ib_conn_error(struct rds_connection *conn, const char *fmt, ...)
1192{
1193 va_list ap;
1194
1195 rds_conn_drop(conn);
1196
1197 va_start(ap, fmt);
1198 vprintk(fmt, ap);
1199 va_end(ap);
1200}
1201