1
2
3
4
5
6
7
8
9
10
11#include <linux/socket.h>
12#include <linux/if_vlan.h>
13#include <linux/random.h>
14#include <linux/workqueue.h>
15#include <net/tcp.h>
16#include <net/sock.h>
17#include <rdma/ib_verbs.h>
18
19#include "smc.h"
20#include "smc_clc.h"
21#include "smc_core.h"
22#include "smc_ib.h"
23#include "smc_wr.h"
24#include "smc_llc.h"
25#include "smc_cdc.h"
26#include "smc_close.h"
27
28#define SMC_LGR_NUM_INCR 256
29#define SMC_LGR_FREE_DELAY (600 * HZ)
30
31static u32 smc_lgr_num;
32
33
34
35
36
37
38
39static void smc_lgr_add_alert_token(struct smc_connection *conn)
40{
41 struct rb_node **link, *parent = NULL;
42 u32 token = conn->alert_token_local;
43
44 link = &conn->lgr->conns_all.rb_node;
45 while (*link) {
46 struct smc_connection *cur = rb_entry(*link,
47 struct smc_connection, alert_node);
48
49 parent = *link;
50 if (cur->alert_token_local > token)
51 link = &parent->rb_left;
52 else
53 link = &parent->rb_right;
54 }
55
56 rb_link_node(&conn->alert_node, parent, link);
57 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);
58}
59
60
61
62
63
64
65static void smc_lgr_register_conn(struct smc_connection *conn)
66{
67 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
68 static atomic_t nexttoken = ATOMIC_INIT(0);
69
70
71
72
73 sock_hold(&smc->sk);
74 while (!conn->alert_token_local) {
75 conn->alert_token_local = atomic_inc_return(&nexttoken);
76 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))
77 conn->alert_token_local = 0;
78 }
79 smc_lgr_add_alert_token(conn);
80 conn->lgr->conns_num++;
81}
82
83
84
85static void __smc_lgr_unregister_conn(struct smc_connection *conn)
86{
87 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
88 struct smc_link_group *lgr = conn->lgr;
89
90 rb_erase(&conn->alert_node, &lgr->conns_all);
91 lgr->conns_num--;
92 conn->alert_token_local = 0;
93 conn->lgr = NULL;
94 sock_put(&smc->sk);
95}
96
97
98
99static void smc_lgr_unregister_conn(struct smc_connection *conn)
100{
101 struct smc_link_group *lgr = conn->lgr;
102 int reduced = 0;
103
104 write_lock_bh(&lgr->conns_lock);
105 if (conn->alert_token_local) {
106 reduced = 1;
107 __smc_lgr_unregister_conn(conn);
108 }
109 write_unlock_bh(&lgr->conns_lock);
110 if (reduced && !lgr->conns_num)
111 schedule_delayed_work(&lgr->free_work, SMC_LGR_FREE_DELAY);
112}
113
114static void smc_lgr_free_work(struct work_struct *work)
115{
116 struct smc_link_group *lgr = container_of(to_delayed_work(work),
117 struct smc_link_group,
118 free_work);
119 bool conns;
120
121 spin_lock_bh(&smc_lgr_list.lock);
122 read_lock_bh(&lgr->conns_lock);
123 conns = RB_EMPTY_ROOT(&lgr->conns_all);
124 read_unlock_bh(&lgr->conns_lock);
125 if (!conns) {
126 spin_unlock_bh(&smc_lgr_list.lock);
127 return;
128 }
129 list_del_init(&lgr->list);
130 spin_unlock_bh(&smc_lgr_list.lock);
131 smc_lgr_free(lgr);
132}
133
134
135static int smc_lgr_create(struct smc_sock *smc, __be32 peer_in_addr,
136 struct smc_ib_device *smcibdev, u8 ibport,
137 char *peer_systemid, unsigned short vlan_id)
138{
139 struct smc_link_group *lgr;
140 struct smc_link *lnk;
141 u8 rndvec[3];
142 int rc = 0;
143 int i;
144
145 lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);
146 if (!lgr) {
147 rc = -ENOMEM;
148 goto out;
149 }
150 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
151 lgr->sync_err = false;
152 lgr->daddr = peer_in_addr;
153 memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN);
154 lgr->vlan_id = vlan_id;
155 rwlock_init(&lgr->sndbufs_lock);
156 rwlock_init(&lgr->rmbs_lock);
157 for (i = 0; i < SMC_RMBE_SIZES; i++) {
158 INIT_LIST_HEAD(&lgr->sndbufs[i]);
159 INIT_LIST_HEAD(&lgr->rmbs[i]);
160 }
161 smc_lgr_num += SMC_LGR_NUM_INCR;
162 memcpy(&lgr->id, (u8 *)&smc_lgr_num, SMC_LGR_ID_SIZE);
163 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);
164 lgr->conns_all = RB_ROOT;
165
166 lnk = &lgr->lnk[SMC_SINGLE_LINK];
167
168 lnk->smcibdev = smcibdev;
169 lnk->ibport = ibport;
170 lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu;
171 if (!smcibdev->initialized)
172 smc_ib_setup_per_ibdev(smcibdev);
173 get_random_bytes(rndvec, sizeof(rndvec));
174 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + (rndvec[2] << 16);
175 rc = smc_wr_alloc_link_mem(lnk);
176 if (rc)
177 goto free_lgr;
178 init_waitqueue_head(&lnk->wr_tx_wait);
179 rc = smc_ib_create_protection_domain(lnk);
180 if (rc)
181 goto free_link_mem;
182 rc = smc_ib_create_queue_pair(lnk);
183 if (rc)
184 goto dealloc_pd;
185 rc = smc_wr_create_link(lnk);
186 if (rc)
187 goto destroy_qp;
188 init_completion(&lnk->llc_confirm);
189 init_completion(&lnk->llc_confirm_resp);
190
191 smc->conn.lgr = lgr;
192 rwlock_init(&lgr->conns_lock);
193 spin_lock_bh(&smc_lgr_list.lock);
194 list_add(&lgr->list, &smc_lgr_list.list);
195 spin_unlock_bh(&smc_lgr_list.lock);
196 return 0;
197
198destroy_qp:
199 smc_ib_destroy_queue_pair(lnk);
200dealloc_pd:
201 smc_ib_dealloc_protection_domain(lnk);
202free_link_mem:
203 smc_wr_free_link_mem(lnk);
204free_lgr:
205 kfree(lgr);
206out:
207 return rc;
208}
209
210static void smc_sndbuf_unuse(struct smc_connection *conn)
211{
212 if (conn->sndbuf_desc) {
213 conn->sndbuf_desc->used = 0;
214 conn->sndbuf_size = 0;
215 }
216}
217
218static void smc_rmb_unuse(struct smc_connection *conn)
219{
220 if (conn->rmb_desc) {
221 conn->rmb_desc->used = 0;
222 conn->rmbe_size = 0;
223 }
224}
225
226
227void smc_conn_free(struct smc_connection *conn)
228{
229 struct smc_link_group *lgr = conn->lgr;
230
231 if (!lgr)
232 return;
233 smc_cdc_tx_dismiss_slots(conn);
234 smc_lgr_unregister_conn(conn);
235 smc_rmb_unuse(conn);
236 smc_sndbuf_unuse(conn);
237}
238
239static void smc_link_clear(struct smc_link *lnk)
240{
241 lnk->peer_qpn = 0;
242 smc_ib_modify_qp_reset(lnk);
243 smc_wr_free_link(lnk);
244 smc_ib_destroy_queue_pair(lnk);
245 smc_ib_dealloc_protection_domain(lnk);
246 smc_wr_free_link_mem(lnk);
247}
248
249static void smc_lgr_free_sndbufs(struct smc_link_group *lgr)
250{
251 struct smc_buf_desc *sndbuf_desc, *bf_desc;
252 int i;
253
254 for (i = 0; i < SMC_RMBE_SIZES; i++) {
255 list_for_each_entry_safe(sndbuf_desc, bf_desc, &lgr->sndbufs[i],
256 list) {
257 list_del(&sndbuf_desc->list);
258 smc_ib_buf_unmap(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
259 smc_uncompress_bufsize(i),
260 sndbuf_desc, DMA_TO_DEVICE);
261 kfree(sndbuf_desc->cpu_addr);
262 kfree(sndbuf_desc);
263 }
264 }
265}
266
267static void smc_lgr_free_rmbs(struct smc_link_group *lgr)
268{
269 struct smc_buf_desc *rmb_desc, *bf_desc;
270 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
271 int i;
272
273 for (i = 0; i < SMC_RMBE_SIZES; i++) {
274 list_for_each_entry_safe(rmb_desc, bf_desc, &lgr->rmbs[i],
275 list) {
276 list_del(&rmb_desc->list);
277 smc_ib_buf_unmap(lnk->smcibdev,
278 smc_uncompress_bufsize(i),
279 rmb_desc, DMA_FROM_DEVICE);
280 kfree(rmb_desc->cpu_addr);
281 kfree(rmb_desc);
282 }
283 }
284}
285
286
287void smc_lgr_free(struct smc_link_group *lgr)
288{
289 smc_lgr_free_rmbs(lgr);
290 smc_lgr_free_sndbufs(lgr);
291 smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]);
292 kfree(lgr);
293}
294
295
296void smc_lgr_terminate(struct smc_link_group *lgr)
297{
298 struct smc_connection *conn;
299 struct smc_sock *smc;
300 struct rb_node *node;
301
302 spin_lock_bh(&smc_lgr_list.lock);
303 if (list_empty(&lgr->list)) {
304
305 spin_unlock_bh(&smc_lgr_list.lock);
306 return;
307 }
308
309 list_del_init(&lgr->list);
310 spin_unlock_bh(&smc_lgr_list.lock);
311
312 write_lock_bh(&lgr->conns_lock);
313 node = rb_first(&lgr->conns_all);
314 while (node) {
315 conn = rb_entry(node, struct smc_connection, alert_node);
316 smc = container_of(conn, struct smc_sock, conn);
317 sock_hold(&smc->sk);
318 __smc_lgr_unregister_conn(conn);
319 schedule_work(&conn->close_work);
320 sock_put(&smc->sk);
321 node = rb_first(&lgr->conns_all);
322 }
323 write_unlock_bh(&lgr->conns_lock);
324}
325
326
327
328
329static int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id)
330{
331 struct dst_entry *dst = sk_dst_get(clcsock->sk);
332 int rc = 0;
333
334 *vlan_id = 0;
335 if (!dst) {
336 rc = -ENOTCONN;
337 goto out;
338 }
339 if (!dst->dev) {
340 rc = -ENODEV;
341 goto out_rel;
342 }
343
344 if (is_vlan_dev(dst->dev))
345 *vlan_id = vlan_dev_vlan_id(dst->dev);
346
347out_rel:
348 dst_release(dst);
349out:
350 return rc;
351}
352
353
354static int smc_link_determine_gid(struct smc_link_group *lgr)
355{
356 struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
357 struct ib_gid_attr gattr;
358 union ib_gid gid;
359 int i;
360
361 if (!lgr->vlan_id) {
362 lnk->gid = lnk->smcibdev->gid[lnk->ibport - 1];
363 return 0;
364 }
365
366 for (i = 0; i < lnk->smcibdev->pattr[lnk->ibport - 1].gid_tbl_len;
367 i++) {
368 if (ib_query_gid(lnk->smcibdev->ibdev, lnk->ibport, i, &gid,
369 &gattr))
370 continue;
371 if (gattr.ndev &&
372 (vlan_dev_vlan_id(gattr.ndev) == lgr->vlan_id)) {
373 lnk->gid = gid;
374 return 0;
375 }
376 }
377 return -ENODEV;
378}
379
380
381int smc_conn_create(struct smc_sock *smc, __be32 peer_in_addr,
382 struct smc_ib_device *smcibdev, u8 ibport,
383 struct smc_clc_msg_local *lcl, int srv_first_contact)
384{
385 struct smc_connection *conn = &smc->conn;
386 struct smc_link_group *lgr;
387 unsigned short vlan_id;
388 enum smc_lgr_role role;
389 int local_contact = SMC_FIRST_CONTACT;
390 int rc = 0;
391
392 role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
393 rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id);
394 if (rc)
395 return rc;
396
397 if ((role == SMC_CLNT) && srv_first_contact)
398
399 goto create;
400
401
402 spin_lock_bh(&smc_lgr_list.lock);
403 list_for_each_entry(lgr, &smc_lgr_list.list, list) {
404 write_lock_bh(&lgr->conns_lock);
405 if (!memcmp(lgr->peer_systemid, lcl->id_for_peer,
406 SMC_SYSTEMID_LEN) &&
407 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid,
408 SMC_GID_SIZE) &&
409 !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac,
410 sizeof(lcl->mac)) &&
411 !lgr->sync_err &&
412 (lgr->role == role) &&
413 (lgr->vlan_id == vlan_id) &&
414 ((role == SMC_CLNT) ||
415 (lgr->conns_num < SMC_RMBS_PER_LGR_MAX))) {
416
417 local_contact = SMC_REUSE_CONTACT;
418 conn->lgr = lgr;
419 smc_lgr_register_conn(conn);
420 write_unlock_bh(&lgr->conns_lock);
421 break;
422 }
423 write_unlock_bh(&lgr->conns_lock);
424 }
425 spin_unlock_bh(&smc_lgr_list.lock);
426
427 if (role == SMC_CLNT && !srv_first_contact &&
428 (local_contact == SMC_FIRST_CONTACT)) {
429
430
431
432
433 return -ENOLINK;
434 }
435
436create:
437 if (local_contact == SMC_FIRST_CONTACT) {
438 rc = smc_lgr_create(smc, peer_in_addr, smcibdev, ibport,
439 lcl->id_for_peer, vlan_id);
440 if (rc)
441 goto out;
442 smc_lgr_register_conn(conn);
443 rc = smc_link_determine_gid(conn->lgr);
444 }
445 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
446 conn->local_tx_ctrl.len = sizeof(struct smc_cdc_msg);
447#ifndef KERNEL_HAS_ATOMIC64
448 spin_lock_init(&conn->acurs_lock);
449#endif
450
451out:
452 return rc ? rc : local_contact;
453}
454
455
456
457
458static inline
459struct smc_buf_desc *smc_sndbuf_get_slot(struct smc_link_group *lgr,
460 int compressed_bufsize)
461{
462 struct smc_buf_desc *sndbuf_slot;
463
464 read_lock_bh(&lgr->sndbufs_lock);
465 list_for_each_entry(sndbuf_slot, &lgr->sndbufs[compressed_bufsize],
466 list) {
467 if (cmpxchg(&sndbuf_slot->used, 0, 1) == 0) {
468 read_unlock_bh(&lgr->sndbufs_lock);
469 return sndbuf_slot;
470 }
471 }
472 read_unlock_bh(&lgr->sndbufs_lock);
473 return NULL;
474}
475
476
477
478
479static inline
480struct smc_buf_desc *smc_rmb_get_slot(struct smc_link_group *lgr,
481 int compressed_bufsize)
482{
483 struct smc_buf_desc *rmb_slot;
484
485 read_lock_bh(&lgr->rmbs_lock);
486 list_for_each_entry(rmb_slot, &lgr->rmbs[compressed_bufsize],
487 list) {
488 if (cmpxchg(&rmb_slot->used, 0, 1) == 0) {
489 read_unlock_bh(&lgr->rmbs_lock);
490 return rmb_slot;
491 }
492 }
493 read_unlock_bh(&lgr->rmbs_lock);
494 return NULL;
495}
496
497
498
499
500
501static inline int smc_rmb_wnd_update_limit(int rmbe_size)
502{
503 return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
504}
505
506
507int smc_sndbuf_create(struct smc_sock *smc)
508{
509 struct smc_connection *conn = &smc->conn;
510 struct smc_link_group *lgr = conn->lgr;
511 int tmp_bufsize, tmp_bufsize_short;
512 struct smc_buf_desc *sndbuf_desc;
513 int rc;
514
515
516 for (tmp_bufsize_short = smc_compress_bufsize(smc->sk.sk_sndbuf / 2);
517 tmp_bufsize_short >= 0; tmp_bufsize_short--) {
518 tmp_bufsize = smc_uncompress_bufsize(tmp_bufsize_short);
519
520 sndbuf_desc = smc_sndbuf_get_slot(lgr, tmp_bufsize_short);
521 if (sndbuf_desc) {
522 memset(sndbuf_desc->cpu_addr, 0, tmp_bufsize);
523 break;
524 }
525
526 sndbuf_desc = kzalloc(sizeof(*sndbuf_desc), GFP_KERNEL);
527 if (!sndbuf_desc)
528 break;
529 sndbuf_desc->cpu_addr = kzalloc(tmp_bufsize,
530 GFP_KERNEL | __GFP_NOWARN |
531 __GFP_NOMEMALLOC |
532 __GFP_NORETRY);
533 if (!sndbuf_desc->cpu_addr) {
534 kfree(sndbuf_desc);
535 sndbuf_desc = NULL;
536
537
538
539 continue;
540 }
541 rc = smc_ib_buf_map(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
542 tmp_bufsize, sndbuf_desc,
543 DMA_TO_DEVICE);
544 if (rc) {
545 kfree(sndbuf_desc->cpu_addr);
546 kfree(sndbuf_desc);
547 sndbuf_desc = NULL;
548 continue;
549 }
550 sndbuf_desc->used = 1;
551 write_lock_bh(&lgr->sndbufs_lock);
552 list_add(&sndbuf_desc->list,
553 &lgr->sndbufs[tmp_bufsize_short]);
554 write_unlock_bh(&lgr->sndbufs_lock);
555 break;
556 }
557 if (sndbuf_desc && sndbuf_desc->cpu_addr) {
558 conn->sndbuf_desc = sndbuf_desc;
559 conn->sndbuf_size = tmp_bufsize;
560 smc->sk.sk_sndbuf = tmp_bufsize * 2;
561 atomic_set(&conn->sndbuf_space, tmp_bufsize);
562 return 0;
563 } else {
564 return -ENOMEM;
565 }
566}
567
568
569
570
571
572
573int smc_rmb_create(struct smc_sock *smc)
574{
575 struct smc_connection *conn = &smc->conn;
576 struct smc_link_group *lgr = conn->lgr;
577 int tmp_bufsize, tmp_bufsize_short;
578 struct smc_buf_desc *rmb_desc;
579 int rc;
580
581
582 for (tmp_bufsize_short = smc_compress_bufsize(smc->sk.sk_rcvbuf / 2);
583 tmp_bufsize_short >= 0; tmp_bufsize_short--) {
584 tmp_bufsize = smc_uncompress_bufsize(tmp_bufsize_short);
585
586 rmb_desc = smc_rmb_get_slot(lgr, tmp_bufsize_short);
587 if (rmb_desc) {
588 memset(rmb_desc->cpu_addr, 0, tmp_bufsize);
589 break;
590 }
591
592 rmb_desc = kzalloc(sizeof(*rmb_desc), GFP_KERNEL);
593 if (!rmb_desc)
594 break;
595 rmb_desc->cpu_addr = kzalloc(tmp_bufsize,
596 GFP_KERNEL | __GFP_NOWARN |
597 __GFP_NOMEMALLOC |
598 __GFP_NORETRY);
599 if (!rmb_desc->cpu_addr) {
600 kfree(rmb_desc);
601 rmb_desc = NULL;
602
603
604
605 continue;
606 }
607 rc = smc_ib_buf_map(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
608 tmp_bufsize, rmb_desc,
609 DMA_FROM_DEVICE);
610 if (rc) {
611 kfree(rmb_desc->cpu_addr);
612 kfree(rmb_desc);
613 rmb_desc = NULL;
614 continue;
615 }
616 rmb_desc->rkey[SMC_SINGLE_LINK] =
617 lgr->lnk[SMC_SINGLE_LINK].roce_pd->unsafe_global_rkey;
618 rmb_desc->used = 1;
619 write_lock_bh(&lgr->rmbs_lock);
620 list_add(&rmb_desc->list,
621 &lgr->rmbs[tmp_bufsize_short]);
622 write_unlock_bh(&lgr->rmbs_lock);
623 break;
624 }
625 if (rmb_desc && rmb_desc->cpu_addr) {
626 conn->rmb_desc = rmb_desc;
627 conn->rmbe_size = tmp_bufsize;
628 conn->rmbe_size_short = tmp_bufsize_short;
629 smc->sk.sk_rcvbuf = tmp_bufsize * 2;
630 atomic_set(&conn->bytes_to_rcv, 0);
631 conn->rmbe_update_limit = smc_rmb_wnd_update_limit(tmp_bufsize);
632 return 0;
633 } else {
634 return -ENOMEM;
635 }
636}
637
638static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
639{
640 int i;
641
642 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {
643 if (!test_and_set_bit(i, lgr->rtokens_used_mask))
644 return i;
645 }
646 return -ENOSPC;
647}
648
649
650int smc_rmb_rtoken_handling(struct smc_connection *conn,
651 struct smc_clc_msg_accept_confirm *clc)
652{
653 u64 dma_addr = be64_to_cpu(clc->rmb_dma_addr);
654 struct smc_link_group *lgr = conn->lgr;
655 u32 rkey = ntohl(clc->rmb_rkey);
656 int i;
657
658 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
659 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) &&
660 (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) &&
661 test_bit(i, lgr->rtokens_used_mask)) {
662 conn->rtoken_idx = i;
663 return 0;
664 }
665 }
666 conn->rtoken_idx = smc_rmb_reserve_rtoken_idx(lgr);
667 if (conn->rtoken_idx < 0)
668 return conn->rtoken_idx;
669 lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].rkey = rkey;
670 lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].dma_addr = dma_addr;
671 return 0;
672}
673